(Note that this patch set depends on virtual ISM support, which is under review:
https://lore.kernel.org/netdev/[email protected]/)
This patch set acts as the second part of the new version of [1], the updated
things of this version are listed at the end.
# Background
SMC-D is now used in IBM z with ISM function to optimize network interconnect
for intra-CPC communications. Inspired by this, we try to make SMC-D available
on the non-s390 architecture through a software-implemented virtual ISM device,
that is the loopback-ism device here, to accelerate inter-process or
inter-containers communication within the same OS instance.
# Design
This patch set includes 3 parts:
- Patch #1-#2: some prepare work for loopback-ism.
- Patch #3-#9: implement loopback-ism device.
- Patch #10-#13: memory copy optimization for loopback scenario.
The loopback-ism device is designed as a ISMv2 device and not be limited to
a specific net namespace, ends of both inter-process connection (1/1' in diagram
below) or inter-container connection (2/2' in diagram below) can find the same
available loopback-ism and choose it during the CLC handshake.
Container 1 (ns1) Container 2 (ns2)
+-----------------------------------------+ +-------------------------+
| +-------+ +-------+ +-------+ | | +-------+ |
| | App A | | App B | | App C | | | | App D |<-+ |
| +-------+ +---^---+ +-------+ | | +-------+ |(2') |
| |127.0.0.1 (1')| |192.168.0.11 192.168.0.12| |
| (1)| +--------+ | +--------+ |(2) | | +--------+ +--------+ |
| `-->| lo |-` | eth0 |<-` | | | lo | | eth0 | |
+---------+--|---^-+---+-----|--+---------+ +-+--------+---+-^------+-+
| | | |
Kernel | | | |
+----+-------v---+-----------v----------------------------------+---+----+
| | TCP | |
| | | |
| +--------------------------------------------------------------+ |
| |
| +--------------+ |
| | smc loopback | |
+---------------------------+--------------+-----------------------------+
loopback-ism device creates DMBs (shared memory) for each connection peer.
Since data transfer occurs within the same kernel, the sndbuf of each peer
is only a descriptor and point to the same memory region as peer DMB, so that
the data copy from sndbuf to peer DMB can be avoided in loopback-ism case.
Container 1 (ns1) Container 2 (ns2)
+-----------------------------------------+ +-------------------------+
| +-------+ | | +-------+ |
| | App C |-----+ | | | App D | |
| +-------+ | | | +-^-----+ |
| | | | | |
| (2) | | | (2') | |
| | | | | |
+---------------|-------------------------+ +----------|--------------+
| |
Kernel | |
+---------------|-----------------------------------------|--------------+
| +--------+ +--v-----+ +--------+ +--------+ |
| |dmb_desc| |snd_desc| |dmb_desc| |snd_desc| |
| +-----|--+ +--|-----+ +-----|--+ +--------+ |
| +-----|--+ | +-----|--+ |
| | DMB C | +---------------------------------| DMB D | |
| +--------+ +--------+ |
| |
| +--------------+ |
| | smc loopback | |
+---------------------------+--------------+-----------------------------+
# Benchmark Test
* Test environments:
- VM with Intel Xeon Platinum 8 core 2.50GHz, 16 GiB mem.
- SMC sndbuf/DMB size 1MB.
* Test object:
- TCP: run on TCP loopback.
- domain: run on UNIX domain.
- SMC lo: run on SMC loopback device.
1. ipc-benchmark (see [2])
- ./<foo> -c 1000000 -s 100
TCP SMC-lo
Message
rate (msg/s) 81539 151251(+85.50%)
2. sockperf
- serv: <smc_run> taskset -c <cpu> sockperf sr --tcp
- clnt: <smc_run> taskset -c <cpu> sockperf { tp | pp } --tcp --msg-size={ 64000 for tp | 14 for pp } -i 127.0.0.1 -t 30
TCP SMC-lo
Bandwidth(MBps) 5313.66 8270.51(+55.65%)
Latency(us) 5.806 3.207(-44.76%)
3. nginx/wrk
- serv: <smc_run> nginx
- clnt: <smc_run> wrk -t 8 -c 1000 -d 30 http://127.0.0.1:80
TCP SMC-lo
Requests/s 194641.79 258656.13(+32.89%)
4. redis-benchmark
- serv: <smc_run> redis-server
- clnt: <smc_run> redis-benchmark -h 127.0.0.1 -q -t set,get -n 400000 -c 200 -d 1024
TCP SMC-lo
GET(Requests/s) 85855.34 115640.35(+34.69%)
SET(Requests/s) 86337.15 118203.30(+36.90%)
[1] https://lore.kernel.org/netdev/[email protected]/
[2] https://github.com/goldsborough/ipc-bench
Updated in this version compare to [1]:
- Patch #1: improve the loopback-ism dump, it shows as follows now:
# smcd d
FID Type PCI-ID PCHID InUse #LGs PNET-ID
0000 0 loopback-ism ffff No 0
- Patch #3: introduce the smc_ism_set_v2_capable() helper and set
smc_ism_v2_capable when ISMv2 or virtual ISM is registered,
regardless of whether there is already a device in smcd device list.
- Patch #3: loopback-ism will be added into /sys/devices/virtual/smc/loopback-ism/.
- Patch #8: introduce the runtime switch /sys/devices/virtual/smc/loopback-ism/active
to activate or deactivate the loopback-ism.
- Patch #9: introduce the statistics of loopback-ism by
/sys/devices/virtual/smc/loopback-ism/{{tx|rx}_tytes|dmbs_cnt}.
- Some minor changes and comments improvements.
Wen Gu (13):
net/smc: improve SMC-D device dump for virtual ISM
net/smc: decouple specialized struct from SMC-D DMB registration
net/smc: introduce virtual ISM device loopback-ism
net/smc: implement ID-related operations of loopback-ism
net/smc: implement some unsupported operations of loopback-ism
net/smc: implement DMB-related operations of loopback-ism
net/smc: register loopback-ism into SMC-D device list
net/smc: introduce loopback-ism runtime switch
net/smc: introduce loopback-ism statistics attributes
net/smc: introduce operations to {at|de}tach ghost sndbuf to peer DMB
net/smc: attach or detach ghost sndbuf to peer DMB.
net/smc: adapt cursor update when sndbuf is mapped to peer DMB
net/smc: implement {at|de}tach_dmb interfaces of loopback-ism
drivers/s390/net/ism_drv.c | 2 +-
include/net/smc.h | 6 +-
net/smc/Kconfig | 13 +
net/smc/Makefile | 2 +-
net/smc/af_smc.c | 33 ++-
net/smc/smc_cdc.c | 58 ++++-
net/smc/smc_cdc.h | 1 +
net/smc/smc_core.c | 71 +++++-
net/smc/smc_core.h | 1 +
net/smc/smc_ism.c | 69 +++++-
net/smc/smc_ism.h | 5 +
net/smc/smc_loopback.c | 603 +++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_loopback.h | 80 ++++++
13 files changed, 915 insertions(+), 29 deletions(-)
create mode 100644 net/smc/smc_loopback.c
create mode 100644 net/smc/smc_loopback.h
--
1.8.3.1
The struct 'ism_client' is specialized for s390 platform firmware ISM.
So replace it with 'void' to make SMCD DMB registration helper generic
for both virtual ISM and existing ISM.
Signed-off-by: Wen Gu <[email protected]>
---
drivers/s390/net/ism_drv.c | 2 +-
include/net/smc.h | 4 ++--
net/smc/smc_ism.c | 7 ++-----
3 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index 2c8e964..9b2a529 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -726,7 +726,7 @@ static int smcd_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
}
static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
- struct ism_client *client)
+ void *client)
{
return ism_register_dmb(smcd->priv, (struct ism_dmb *)dmb, client);
}
diff --git a/include/net/smc.h b/include/net/smc.h
index c9dcb30..6273c3a 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -50,7 +50,6 @@ struct smcd_dmb {
#define ISM_ERROR 0xFFFF
struct smcd_dev;
-struct ism_client;
struct smcd_gid {
u64 gid;
@@ -61,7 +60,7 @@ struct smcd_ops {
int (*query_remote_gid)(struct smcd_dev *dev, struct smcd_gid *rgid,
u32 vid_valid, u32 vid);
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
- struct ism_client *client);
+ void *client);
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
int (*add_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*del_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
@@ -81,6 +80,7 @@ struct smcd_ops {
struct smcd_dev {
const struct smcd_ops *ops;
void *priv;
+ void *client;
struct list_head list;
spinlock_t lock;
struct smc_connection **conn;
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 66bcfdd..fb1837d 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -222,7 +222,6 @@ int smc_ism_unregister_dmb(struct smcd_dev *smcd, struct smc_buf_desc *dmb_desc)
int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
struct smc_buf_desc *dmb_desc)
{
-#if IS_ENABLED(CONFIG_ISM)
struct smcd_dmb dmb;
int rc;
@@ -231,7 +230,7 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
dmb.sba_idx = dmb_desc->sba_idx;
dmb.vlan_id = lgr->vlan_id;
dmb.rgid = lgr->peer_gid.gid;
- rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, &smc_ism_client);
+ rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, lgr->smcd->client);
if (!rc) {
dmb_desc->sba_idx = dmb.sba_idx;
dmb_desc->token = dmb.dmb_tok;
@@ -240,9 +239,6 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
dmb_desc->len = dmb.dmb_len;
}
return rc;
-#else
- return 0;
-#endif
}
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
@@ -453,6 +449,7 @@ static void smcd_register_dev(struct ism_dev *ism)
if (!smcd)
return;
smcd->priv = ism;
+ smcd->client = &smc_ism_client;
ism_set_priv(ism, &smc_ism_client, smcd);
if (smc_pnetid_by_dev_port(&ism->pdev->dev, 0, smcd->pnetid))
smc_pnetid_by_table_smcd(smcd);
--
1.8.3.1
After loopback-ism device gets ready, add it to the SMC-D device list as
an ISMv2 device.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_ism.c | 11 +++++++----
net/smc/smc_ism.h | 1 +
net/smc/smc_loopback.c | 20 +++++++++++++-------
3 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index fb1837d..4065ebd 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -91,6 +91,11 @@ bool smc_ism_is_v2_capable(void)
return smc_ism_v2_capable;
}
+void smc_ism_set_v2_capable(void)
+{
+ smc_ism_v2_capable = true;
+}
+
/* Set a connection using this DMBE. */
void smc_ism_set_conn(struct smc_connection *conn)
{
@@ -454,11 +459,9 @@ static void smcd_register_dev(struct ism_dev *ism)
if (smc_pnetid_by_dev_port(&ism->pdev->dev, 0, smcd->pnetid))
smc_pnetid_by_table_smcd(smcd);
+ if (smcd->ops->supports_v2())
+ smc_ism_set_v2_capable();
mutex_lock(&smcd_dev_list.mutex);
- if (list_empty(&smcd_dev_list.list)) {
- if (smcd->ops->supports_v2())
- smc_ism_v2_capable = true;
- }
/* sort list: devices without pnetid before devices with pnetid */
if (smcd->pnetid[0])
list_add_tail(&smcd->list, &smcd_dev_list.list);
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index ffff40c..6903cd5 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -52,6 +52,7 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int buf_size,
void smc_ism_get_system_eid(u8 **eid);
u16 smc_ism_get_chid(struct smcd_dev *dev);
bool smc_ism_is_v2_capable(void);
+void smc_ism_set_v2_capable(void);
int smc_ism_init(void);
void smc_ism_exit(void);
int smcd_nl_get_device(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index ee4f112..83bc9a7 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -273,10 +273,12 @@ static int smcd_lo_register_dev(struct smc_lo_dev *ldev)
return -ENOMEM;
ldev->smcd = smcd;
smcd->priv = ldev;
-
- /* TODO:
- * register loopback-ism to smcd_dev list.
- */
+ smc_ism_set_v2_capable();
+ mutex_lock(&smcd_dev_list.mutex);
+ list_add(&smcd->list, &smcd_dev_list.list);
+ mutex_unlock(&smcd_dev_list.mutex);
+ pr_warn_ratelimited("smc: adding smcd device %s\n",
+ smc_lo_dev_name);
return 0;
}
@@ -284,9 +286,13 @@ static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
{
struct smcd_dev *smcd = ldev->smcd;
- /* TODO:
- * unregister loopback-ism from smcd_dev list.
- */
+ pr_warn_ratelimited("smc: removing smcd device %s\n",
+ smc_lo_dev_name);
+ smcd->going_away = 1;
+ smc_smcd_terminate_all(smcd);
+ mutex_lock(&smcd_dev_list.mutex);
+ list_del_init(&smcd->list);
+ mutex_unlock(&smcd_dev_list.mutex);
kfree(smcd->conn);
kfree(smcd);
}
--
1.8.3.1
This implements GID and CHID related operations of loopback-ism device.
loopback-ism acts as an ISMv2. It's GID is generated randomly by UUIDv4
algorithm and CHID is reserved 0xFFFF.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++----
net/smc/smc_loopback.h | 3 +++
2 files changed, 60 insertions(+), 5 deletions(-)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 149ccaa..39e8c8d 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -19,12 +19,63 @@
#include "smc_loopback.h"
#if IS_ENABLED(CONFIG_SMC_LO)
+#define SMC_LO_V2_CAPABLE 0x1 /* loopback-ism acts as ISMv2 */
+
static const char smc_lo_dev_name[] = "loopback-ism";
static struct smc_lo_dev *lo_dev;
static struct class *smc_class;
+static void smc_lo_generate_id(struct smc_lo_dev *ldev)
+{
+ struct smcd_gid *lgid = &ldev->local_gid;
+ uuid_t uuid;
+
+ uuid_gen(&uuid);
+ memcpy(&lgid->gid, &uuid, sizeof(lgid->gid));
+ memcpy(&lgid->gid_ext, (u8 *)&uuid + sizeof(lgid->gid),
+ sizeof(lgid->gid_ext));
+
+ ldev->chid = SMC_LO_CHID;
+}
+
+static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
+ u32 vid_valid, u32 vid)
+{
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* rgid should equal to lgid in loopback situation */
+ if (!ldev || rgid->gid != ldev->local_gid.gid ||
+ rgid->gid_ext != ldev->local_gid.gid_ext)
+ return -ENETUNREACH;
+ return 0;
+}
+
+static int smc_lo_supports_v2(void)
+{
+ return SMC_LO_V2_CAPABLE;
+}
+
+static void smc_lo_get_local_gid(struct smcd_dev *smcd,
+ struct smcd_gid *smcd_gid)
+{
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ smcd_gid->gid = ldev->local_gid.gid;
+ smcd_gid->gid_ext = ldev->local_gid.gid_ext;
+}
+
+static u16 smc_lo_get_chid(struct smcd_dev *smcd)
+{
+ return ((struct smc_lo_dev *)smcd->priv)->chid;
+}
+
+static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
+{
+ return &((struct smc_lo_dev *)smcd->priv)->dev;
+}
+
static const struct smcd_ops lo_ops = {
- .query_remote_gid = NULL,
+ .query_remote_gid = smc_lo_query_rgid,
.register_dmb = NULL,
.unregister_dmb = NULL,
.add_vlan_id = NULL,
@@ -33,10 +84,10 @@
.reset_vlan_required = NULL,
.signal_event = NULL,
.move_data = NULL,
- .supports_v2 = NULL,
- .get_local_gid = NULL,
- .get_chid = NULL,
- .get_dev = NULL,
+ .supports_v2 = smc_lo_supports_v2,
+ .get_local_gid = smc_lo_get_local_gid,
+ .get_chid = smc_lo_get_chid,
+ .get_dev = smc_lo_get_dev,
};
static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops,
@@ -96,6 +147,7 @@ static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
static int smc_lo_dev_init(struct smc_lo_dev *ldev)
{
+ smc_lo_generate_id(ldev);
return smcd_lo_register_dev(ldev);
}
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index a7c418b..73eed6a 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -20,10 +20,13 @@
#if IS_ENABLED(CONFIG_SMC_LO)
#define SMC_LO_MAX_DMBS 5000
+#define SMC_LO_CHID 0xFFFF
struct smc_lo_dev {
struct smcd_dev *smcd;
struct device dev;
+ u16 chid;
+ struct smcd_gid local_gid;
};
#endif
--
1.8.3.1
This introduces some statistics attributes of loopback-ism. They can be
read from /sys/devices/virtual/smc/loopback-ism/{{tx|rx}_tytes|dmbs_cnt}.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++-
net/smc/smc_loopback.h | 26 ++++++++++++++++
2 files changed, 105 insertions(+), 1 deletion(-)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 04f8612..057ea6a 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -31,6 +31,67 @@
static int smcd_lo_register_dev(struct smc_lo_dev *ldev);
static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev);
+static void smc_lo_clear_stats(struct smc_lo_dev *ldev)
+{
+ struct smc_lo_dev_stats64 *tmp;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ tmp = per_cpu_ptr(ldev->stats, cpu);
+ tmp->tx_bytes = 0;
+ tmp->rx_bytes = 0;
+ }
+}
+
+static void smc_lo_get_stats(struct smc_lo_dev *ldev,
+ struct smc_lo_dev_stats64 *stats)
+{
+ int size, cpu, i;
+ u64 *src, *sum;
+
+ memset(stats, 0, sizeof(*stats));
+ size = sizeof(*stats) / sizeof(u64);
+ for_each_possible_cpu(cpu) {
+ src = (u64 *)per_cpu_ptr(ldev->stats, cpu);
+ sum = (u64 *)stats;
+ for (i = 0; i < size; i++)
+ *(sum++) += *(src++);
+ }
+}
+
+static ssize_t smc_lo_show_stats(struct device *dev,
+ struct device_attribute *attr,
+ char *buf, unsigned long offset)
+{
+ struct smc_lo_dev *ldev =
+ container_of(dev, struct smc_lo_dev, dev);
+ struct smc_lo_dev_stats64 stats;
+ ssize_t ret = -EINVAL;
+
+ if (WARN_ON(offset > sizeof(struct smc_lo_dev_stats64) ||
+ offset % sizeof(u64) != 0))
+ goto out;
+
+ smc_lo_get_stats(ldev, &stats);
+ ret = sysfs_emit(buf, "%llu\n", *(u64 *)(((u8 *)&stats) + offset));
+out:
+ return ret;
+}
+
+/* generate a read-only statistics attribute */
+#define SMC_LO_DEVICE_ATTR_RO(name) \
+static ssize_t name##_show(struct device *dev, \
+ struct device_attribute *attr, char *buf) \
+{ \
+ return smc_lo_show_stats(dev, attr, buf, \
+ offsetof(struct smc_lo_dev_stats64, name)); \
+} \
+static DEVICE_ATTR_RO(name)
+
+SMC_LO_DEVICE_ATTR_RO(rx_bytes);
+SMC_LO_DEVICE_ATTR_RO(tx_bytes);
+SMC_LO_DEVICE_ATTR_RO(dmbs_cnt);
+
static ssize_t active_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -68,6 +129,9 @@ static ssize_t active_store(struct device *dev,
static DEVICE_ATTR_RW(active);
static struct attribute *smc_lo_attrs[] = {
&dev_attr_active.attr,
+ &dev_attr_rx_bytes.attr,
+ &dev_attr_tx_bytes.attr,
+ &dev_attr_dmbs_cnt.attr,
NULL
};
@@ -147,6 +211,7 @@ static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
}
hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
write_unlock(&ldev->dmb_ht_lock);
+ SMC_LO_STAT_DMBS_INC(ldev);
dmb->sba_idx = dmb_node->sba_idx;
dmb->dmb_tok = dmb_node->token;
@@ -186,6 +251,7 @@ static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
vfree(dmb_node->cpu_addr);
kfree(dmb_node);
+ SMC_LO_STAT_DMBS_DEC(ldev);
return 0;
}
@@ -237,13 +303,16 @@ static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok,
read_unlock(&ldev->dmb_ht_lock);
memcpy((char *)rmb_node->cpu_addr + offset, data, size);
+ SMC_LO_STAT_TX_BYTES(ldev, size);
if (sf) {
struct smc_connection *conn =
smcd->conn[rmb_node->sba_idx];
- if (conn && !conn->killed)
+ if (conn && !conn->killed) {
+ SMC_LO_STAT_RX_BYTES(ldev, size);
smcd_cdc_rx_handler(conn);
+ }
}
return 0;
}
@@ -349,6 +418,7 @@ static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
mutex_unlock(&smcd_dev_list.mutex);
kfree(smcd->conn);
kfree(smcd);
+ smc_lo_clear_stats(ldev);
}
static int smc_lo_dev_init(struct smc_lo_dev *ldev)
@@ -369,6 +439,7 @@ static void smc_lo_dev_release(struct device *dev)
struct smc_lo_dev *ldev =
container_of(dev, struct smc_lo_dev, dev);
+ free_percpu(ldev->stats);
kfree(ldev);
}
@@ -387,6 +458,13 @@ static int smc_lo_dev_probe(void)
goto destroy_class;
}
+ ldev->stats = alloc_percpu(struct smc_lo_dev_stats64);
+ if (!ldev->stats) {
+ ret = -ENOMEM;
+ kfree(ldev);
+ goto destroy_class;
+ }
+
ldev->dev.parent = NULL;
ldev->dev.class = smc_class;
ldev->dev.groups = smc_lo_attr_groups;
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index 0cc2f83..ad0feaa 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -32,16 +32,42 @@ struct smc_lo_dmb_node {
dma_addr_t dma_addr;
};
+struct smc_lo_dev_stats64 {
+ __u64 rx_bytes;
+ __u64 tx_bytes;
+ __u64 dmbs_cnt;
+};
+
struct smc_lo_dev {
struct smcd_dev *smcd;
struct device dev;
u8 active;
u16 chid;
struct smcd_gid local_gid;
+ struct smc_lo_dev_stats64 __percpu *stats;
rwlock_t dmb_ht_lock;
DECLARE_BITMAP(sba_idx_mask, SMC_LO_MAX_DMBS);
DECLARE_HASHTABLE(dmb_ht, SMC_LO_DMBS_HASH_BITS);
};
+
+#define SMC_LO_STAT_SUB(ldev, key, val) \
+do { \
+ struct smc_lo_dev_stats64 *_stats = (ldev)->stats; \
+ this_cpu_add((*(_stats)).key, val); \
+} \
+while (0)
+
+#define SMC_LO_STAT_RX_BYTES(ldev, val) \
+ SMC_LO_STAT_SUB(ldev, rx_bytes, val)
+
+#define SMC_LO_STAT_TX_BYTES(ldev, val) \
+ SMC_LO_STAT_SUB(ldev, tx_bytes, val)
+
+#define SMC_LO_STAT_DMBS_INC(ldev) \
+ SMC_LO_STAT_SUB(ldev, dmbs_cnt, 1)
+
+#define SMC_LO_STAT_DMBS_DEC(ldev) \
+ SMC_LO_STAT_SUB(ldev, dmbs_cnt, -1)
#endif
int smc_loopback_init(void);
--
1.8.3.1
This introduces a kind of virtual ISM device loopback-ism for SMCDv2.1.
loopback-ism is implemented by software and serves inter-process or
inter-container SMC communication in the same OS instance. It is created
during SMC module loading and destroyed upon unloading. The support for
loopback-ism can be configured via CONFIG_SMC_LO.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/Kconfig | 13 ++++
net/smc/Makefile | 2 +-
net/smc/af_smc.c | 12 +++-
net/smc/smc_loopback.c | 181 +++++++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_loopback.h | 33 +++++++++
5 files changed, 239 insertions(+), 2 deletions(-)
create mode 100644 net/smc/smc_loopback.c
create mode 100644 net/smc/smc_loopback.h
diff --git a/net/smc/Kconfig b/net/smc/Kconfig
index 746be39..e191f78 100644
--- a/net/smc/Kconfig
+++ b/net/smc/Kconfig
@@ -20,3 +20,16 @@ config SMC_DIAG
smcss.
if unsure, say Y.
+
+config SMC_LO
+ bool "SMC_LO: virtual ISM loopback-ism for SMC"
+ depends on SMC
+ default n
+ help
+ SMC_LO provides a kind of virtual ISM device called loopback-ism
+ for SMCD to upgrade AF_INET TCP connections whose ends share the
+ same kernel.
+ loopback-ism is a software implemented device that does not depend
+ on a specific architecture or hardware.
+
+ if unsure, say N.
diff --git a/net/smc/Makefile b/net/smc/Makefile
index 875efcd..a8c3711 100644
--- a/net/smc/Makefile
+++ b/net/smc/Makefile
@@ -4,5 +4,5 @@ obj-$(CONFIG_SMC) += smc.o
obj-$(CONFIG_SMC_DIAG) += smc_diag.o
smc-y := af_smc.o smc_pnet.o smc_ib.o smc_clc.o smc_core.o smc_wr.o smc_llc.o
smc-y += smc_cdc.o smc_tx.o smc_rx.o smc_close.o smc_ism.o smc_netlink.o smc_stats.o
-smc-y += smc_tracepoint.o
+smc-y += smc_tracepoint.o smc_loopback.o
smc-$(CONFIG_SYSCTL) += smc_sysctl.o
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 9cdc974..6c0e381 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -53,6 +53,7 @@
#include "smc_stats.h"
#include "smc_tracepoint.h"
#include "smc_sysctl.h"
+#include "smc_loopback.h"
static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
* creation on server
@@ -3574,15 +3575,23 @@ static int __init smc_init(void)
goto out_sock;
}
+ rc = smc_loopback_init();
+ if (rc) {
+ pr_err("%s: smc_loopback_init fails with %d\n", __func__, rc);
+ goto out_ib;
+ }
+
rc = tcp_register_ulp(&smc_ulp_ops);
if (rc) {
pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
- goto out_ib;
+ goto out_lo;
}
static_branch_enable(&tcp_have_smc);
return 0;
+out_lo:
+ smc_loopback_exit();
out_ib:
smc_ib_unregister_client();
out_sock:
@@ -3620,6 +3629,7 @@ static void __exit smc_exit(void)
tcp_unregister_ulp(&smc_ulp_ops);
sock_unregister(PF_SMC);
smc_core_exit();
+ smc_loopback_exit();
smc_ib_unregister_client();
smc_ism_exit();
destroy_workqueue(smc_close_wq);
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
new file mode 100644
index 0000000..149ccaa
--- /dev/null
+++ b/net/smc/smc_loopback.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shared Memory Communications Direct over loopback-ism device.
+ *
+ * Provide a SMC-D loopback-ism device.
+ *
+ * Copyright (c) 2023, Alibaba Inc.
+ *
+ * Author: Wen Gu <[email protected]>
+ * Tony Lu <[email protected]>
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/types.h>
+#include <net/smc.h>
+
+#include "smc_ism.h"
+#include "smc_loopback.h"
+
+#if IS_ENABLED(CONFIG_SMC_LO)
+static const char smc_lo_dev_name[] = "loopback-ism";
+static struct smc_lo_dev *lo_dev;
+static struct class *smc_class;
+
+static const struct smcd_ops lo_ops = {
+ .query_remote_gid = NULL,
+ .register_dmb = NULL,
+ .unregister_dmb = NULL,
+ .add_vlan_id = NULL,
+ .del_vlan_id = NULL,
+ .set_vlan_required = NULL,
+ .reset_vlan_required = NULL,
+ .signal_event = NULL,
+ .move_data = NULL,
+ .supports_v2 = NULL,
+ .get_local_gid = NULL,
+ .get_chid = NULL,
+ .get_dev = NULL,
+};
+
+static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops,
+ int max_dmbs)
+{
+ struct smcd_dev *smcd;
+
+ smcd = kzalloc(sizeof(*smcd), GFP_KERNEL);
+ if (!smcd)
+ return NULL;
+
+ smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *),
+ GFP_KERNEL);
+ if (!smcd->conn)
+ goto out_smcd;
+
+ smcd->ops = ops;
+
+ spin_lock_init(&smcd->lock);
+ spin_lock_init(&smcd->lgr_lock);
+ INIT_LIST_HEAD(&smcd->vlan);
+ INIT_LIST_HEAD(&smcd->lgr_list);
+ init_waitqueue_head(&smcd->lgrs_deleted);
+ return smcd;
+
+out_smcd:
+ kfree(smcd);
+ return NULL;
+}
+
+static int smcd_lo_register_dev(struct smc_lo_dev *ldev)
+{
+ struct smcd_dev *smcd;
+
+ smcd = smcd_lo_alloc_dev(&lo_ops, SMC_LO_MAX_DMBS);
+ if (!smcd)
+ return -ENOMEM;
+ ldev->smcd = smcd;
+ smcd->priv = ldev;
+
+ /* TODO:
+ * register loopback-ism to smcd_dev list.
+ */
+ return 0;
+}
+
+static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
+{
+ struct smcd_dev *smcd = ldev->smcd;
+
+ /* TODO:
+ * unregister loopback-ism from smcd_dev list.
+ */
+ kfree(smcd->conn);
+ kfree(smcd);
+}
+
+static int smc_lo_dev_init(struct smc_lo_dev *ldev)
+{
+ return smcd_lo_register_dev(ldev);
+}
+
+static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
+{
+ smcd_lo_unregister_dev(ldev);
+}
+
+static void smc_lo_dev_release(struct device *dev)
+{
+ struct smc_lo_dev *ldev =
+ container_of(dev, struct smc_lo_dev, dev);
+
+ kfree(ldev);
+}
+
+static int smc_lo_dev_probe(void)
+{
+ struct smc_lo_dev *ldev;
+ int ret;
+
+ smc_class = class_create("smc");
+ if (IS_ERR(smc_class))
+ return PTR_ERR(smc_class);
+
+ ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
+ if (!ldev) {
+ ret = -ENOMEM;
+ goto destroy_class;
+ }
+
+ ldev->dev.parent = NULL;
+ ldev->dev.class = smc_class;
+ ldev->dev.release = smc_lo_dev_release;
+ device_initialize(&ldev->dev);
+ dev_set_name(&ldev->dev, smc_lo_dev_name);
+ ret = device_add(&ldev->dev);
+ if (ret)
+ goto free_dev;
+
+ ret = smc_lo_dev_init(ldev);
+ if (ret)
+ goto del_dev;
+
+ lo_dev = ldev; /* global loopback device */
+ return 0;
+
+del_dev:
+ device_del(&ldev->dev);
+free_dev:
+ put_device(&ldev->dev);
+destroy_class:
+ class_destroy(smc_class);
+ return ret;
+}
+
+static void smc_lo_dev_remove(void)
+{
+ if (!lo_dev)
+ return;
+
+ smc_lo_dev_exit(lo_dev);
+ device_del(&lo_dev->dev); /* device_add in smc_lo_dev_probe */
+ put_device(&lo_dev->dev); /* device_initialize in smc_lo_dev_probe */
+ class_destroy(smc_class);
+}
+#endif
+
+int smc_loopback_init(void)
+{
+#if IS_ENABLED(CONFIG_SMC_LO)
+ return smc_lo_dev_probe();
+#else
+ return 0;
+#endif
+}
+
+void smc_loopback_exit(void)
+{
+#if IS_ENABLED(CONFIG_SMC_LO)
+ smc_lo_dev_remove();
+#endif
+}
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
new file mode 100644
index 0000000..a7c418b
--- /dev/null
+++ b/net/smc/smc_loopback.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shared Memory Communications Direct over loopback-ism device.
+ *
+ * Provide a SMC-D loopback-ism device.
+ *
+ * Copyright (c) 2023, Alibaba Inc.
+ *
+ * Author: Wen Gu <[email protected]>
+ * Tony Lu <[email protected]>
+ *
+ */
+
+#ifndef _SMC_LOOPBACK_H
+#define _SMC_LOOPBACK_H
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <net/smc.h>
+
+#if IS_ENABLED(CONFIG_SMC_LO)
+#define SMC_LO_MAX_DMBS 5000
+
+struct smc_lo_dev {
+ struct smcd_dev *smcd;
+ struct device dev;
+};
+#endif
+
+int smc_loopback_init(void);
+void smc_loopback_exit(void);
+
+#endif /* _SMC_LOOPBACK_H */
--
1.8.3.1
vlan operations are not supported currently since the need for vlan in
loopback situation does not seem to be strong.
signal_event operation is not supported since no event now needs to be
processed by loopback-ism device.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 39e8c8d..349a969 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -50,6 +50,32 @@ static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
return 0;
}
+static int smc_lo_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
+{
+ return -EOPNOTSUPP;
+}
+
+static int smc_lo_del_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
+{
+ return -EOPNOTSUPP;
+}
+
+static int smc_lo_set_vlan_required(struct smcd_dev *smcd)
+{
+ return -EOPNOTSUPP;
+}
+
+static int smc_lo_reset_vlan_required(struct smcd_dev *smcd)
+{
+ return -EOPNOTSUPP;
+}
+
+static int smc_lo_signal_event(struct smcd_dev *dev, struct smcd_gid *rgid,
+ u32 trigger_irq, u32 event_code, u64 info)
+{
+ return 0;
+}
+
static int smc_lo_supports_v2(void)
{
return SMC_LO_V2_CAPABLE;
@@ -78,11 +104,11 @@ static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
.query_remote_gid = smc_lo_query_rgid,
.register_dmb = NULL,
.unregister_dmb = NULL,
- .add_vlan_id = NULL,
- .del_vlan_id = NULL,
- .set_vlan_required = NULL,
- .reset_vlan_required = NULL,
- .signal_event = NULL,
+ .add_vlan_id = smc_lo_add_vlan_id,
+ .del_vlan_id = smc_lo_del_vlan_id,
+ .set_vlan_required = smc_lo_set_vlan_required,
+ .reset_vlan_required = smc_lo_reset_vlan_required,
+ .signal_event = smc_lo_signal_event,
.move_data = NULL,
.supports_v2 = smc_lo_supports_v2,
.get_local_gid = smc_lo_get_local_gid,
--
1.8.3.1
This provides a runtime switch to activate or deactivate loopback-ism
device by echo {1|0} > /sys/devices/virtual/smc/loopback-ism/active. It
will trigger the registration or removal of loopback-ism from the SMC-D
device list.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_loopback.h | 1 +
2 files changed, 56 insertions(+)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 83bc9a7..04f8612 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -28,6 +28,58 @@
static struct smc_lo_dev *lo_dev;
static struct class *smc_class;
+static int smcd_lo_register_dev(struct smc_lo_dev *ldev);
+static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev);
+
+static ssize_t active_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct smc_lo_dev *ldev =
+ container_of(dev, struct smc_lo_dev, dev);
+
+ return sysfs_emit(buf, "%d\n", ldev->active);
+}
+
+static ssize_t active_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct smc_lo_dev *ldev =
+ container_of(dev, struct smc_lo_dev, dev);
+ bool active;
+ int ret;
+
+ ret = kstrtobool(buf, &active);
+ if (ret)
+ return ret;
+
+ if (active && !ldev->active) {
+ /* activate loopback-ism */
+ ret = smcd_lo_register_dev(ldev);
+ if (ret)
+ return ret;
+ } else if (!active && ldev->active) {
+ /* deactivate loopback-ism */
+ smcd_lo_unregister_dev(ldev);
+ }
+
+ return count;
+}
+static DEVICE_ATTR_RW(active);
+static struct attribute *smc_lo_attrs[] = {
+ &dev_attr_active.attr,
+ NULL
+};
+
+static struct attribute_group smc_lo_attr_group = {
+ .attrs = smc_lo_attrs,
+};
+
+static const struct attribute_group *smc_lo_attr_groups[] = {
+ &smc_lo_attr_group,
+ NULL,
+};
+
static void smc_lo_generate_id(struct smc_lo_dev *ldev)
{
struct smcd_gid *lgid = &ldev->local_gid;
@@ -277,6 +329,7 @@ static int smcd_lo_register_dev(struct smc_lo_dev *ldev)
mutex_lock(&smcd_dev_list.mutex);
list_add(&smcd->list, &smcd_dev_list.list);
mutex_unlock(&smcd_dev_list.mutex);
+ ldev->active = 1;
pr_warn_ratelimited("smc: adding smcd device %s\n",
smc_lo_dev_name);
return 0;
@@ -288,6 +341,7 @@ static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
pr_warn_ratelimited("smc: removing smcd device %s\n",
smc_lo_dev_name);
+ ldev->active = 0;
smcd->going_away = 1;
smc_smcd_terminate_all(smcd);
mutex_lock(&smcd_dev_list.mutex);
@@ -335,6 +389,7 @@ static int smc_lo_dev_probe(void)
ldev->dev.parent = NULL;
ldev->dev.class = smc_class;
+ ldev->dev.groups = smc_lo_attr_groups;
ldev->dev.release = smc_lo_dev_release;
device_initialize(&ldev->dev);
dev_set_name(&ldev->dev, smc_lo_dev_name);
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index 3d5d2f5..0cc2f83 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -35,6 +35,7 @@ struct smc_lo_dmb_node {
struct smc_lo_dev {
struct smcd_dev *smcd;
struct device dev;
+ u8 active;
u16 chid;
struct smcd_gid local_gid;
rwlock_t dmb_ht_lock;
--
1.8.3.1
The ghost sndbuf descriptor will be created and attached to peer DMB
once peer token is obtained and it will be detach and freed when the
connection is freed.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/af_smc.c | 21 ++++++++++++++++
net/smc/smc_core.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
net/smc/smc_core.h | 1 +
3 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 6c0e381..838cf9c 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -1449,6 +1449,14 @@ static int smc_connect_ism(struct smc_sock *smc,
}
smc_conn_save_peer_info(smc, aclc);
+
+ if (smc_ism_dmb_mappable(smc->conn.lgr->smcd)) {
+ rc = smcd_buf_attach(smc);
+ if (rc) {
+ rc = SMC_CLC_DECL_MEM; /* try to fallback */
+ goto connect_abort;
+ }
+ }
smc_close_init(smc);
smc_rx_init(smc);
smc_tx_init(smc);
@@ -2559,6 +2567,19 @@ static void smc_listen_work(struct work_struct *work)
mutex_unlock(&smc_server_lgr_pending);
}
smc_conn_save_peer_info(new_smc, cclc);
+
+ if (ini->is_smcd &&
+ smc_ism_dmb_mappable(new_smc->conn.lgr->smcd)) {
+ rc = smcd_buf_attach(new_smc);
+ if (rc)
+ /* SMC-D client has completed the CLC process,
+ * so server can not send decline any more.
+ * This requires the rc returned here no larger
+ * than 0.
+ */
+ goto out_decl;
+ }
+
smc_listen_out_connected(new_smc);
SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
goto out_free;
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 95cc954..4a76e38 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -1149,6 +1149,20 @@ static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb,
}
}
+static void smcd_buf_detach(struct smc_connection *conn)
+{
+ struct smcd_dev *smcd = conn->lgr->smcd;
+ u64 peer_token = conn->peer_token;
+
+ if (!conn->sndbuf_desc)
+ return;
+
+ smc_ism_detach_dmb(smcd, peer_token);
+
+ kfree(conn->sndbuf_desc);
+ conn->sndbuf_desc = NULL;
+}
+
static void smc_buf_unuse(struct smc_connection *conn,
struct smc_link_group *lgr)
{
@@ -1192,6 +1206,8 @@ void smc_conn_free(struct smc_connection *conn)
if (lgr->is_smcd) {
if (!list_empty(&lgr->list))
smc_ism_unset_conn(conn);
+ if (smc_ism_dmb_mappable(lgr->smcd))
+ smcd_buf_detach(conn);
tasklet_kill(&conn->rx_tsklet);
} else {
smc_cdc_wait_pend_tx_wr(conn);
@@ -1445,6 +1461,8 @@ static void smc_conn_kill(struct smc_connection *conn, bool soft)
smc_sk_wake_ups(smc);
if (conn->lgr->is_smcd) {
smc_ism_unset_conn(conn);
+ if (smc_ism_dmb_mappable(conn->lgr->smcd))
+ smcd_buf_detach(conn);
if (soft)
tasklet_kill(&conn->rx_tsklet);
else
@@ -2455,15 +2473,23 @@ void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
*/
int smc_buf_create(struct smc_sock *smc, bool is_smcd)
{
+ bool sndbuf_created = false;
int rc;
/* create send buffer */
+ if (is_smcd &&
+ smc_ism_dmb_mappable(smc->conn.lgr->smcd))
+ goto create_rmb;
+
rc = __smc_buf_create(smc, is_smcd, false);
if (rc)
return rc;
+ sndbuf_created = true;
+
+create_rmb:
/* create rmb */
rc = __smc_buf_create(smc, is_smcd, true);
- if (rc) {
+ if (rc && sndbuf_created) {
down_write(&smc->conn.lgr->sndbufs_lock);
list_del(&smc->conn.sndbuf_desc->list);
up_write(&smc->conn.lgr->sndbufs_lock);
@@ -2473,6 +2499,49 @@ int smc_buf_create(struct smc_sock *smc, bool is_smcd)
return rc;
}
+int smcd_buf_attach(struct smc_sock *smc)
+{
+ struct smc_connection *conn = &smc->conn;
+ struct smcd_dev *smcd = conn->lgr->smcd;
+ u64 peer_token = conn->peer_token;
+ struct smc_buf_desc *buf_desc;
+ int rc;
+
+ buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
+ if (!buf_desc)
+ return -ENOMEM;
+
+ /* The ghost sndbuf_desc describes the same memory region as
+ * peer RMB. Its lifecycle is consistent with the connection's
+ * and it will be freed with the connections instead of the
+ * link group.
+ */
+ rc = smc_ism_attach_dmb(smcd, peer_token, buf_desc);
+ if (rc)
+ goto free;
+
+ smc->sk.sk_sndbuf = buf_desc->len;
+ buf_desc->cpu_addr =
+ (u8 *)buf_desc->cpu_addr + sizeof(struct smcd_cdc_msg);
+ buf_desc->len -= sizeof(struct smcd_cdc_msg);
+ conn->sndbuf_desc = buf_desc;
+ conn->sndbuf_desc->used = 1;
+ atomic_set(&conn->sndbuf_space, conn->sndbuf_desc->len);
+ return 0;
+
+free:
+ if (conn->rmb_desc) {
+ /* free local RMB as well */
+ down_write(&conn->lgr->rmbs_lock);
+ list_del(&conn->rmb_desc->list);
+ up_write(&conn->lgr->rmbs_lock);
+ smc_buf_free(conn->lgr, true, conn->rmb_desc);
+ conn->rmb_desc = NULL;
+ }
+ kfree(buf_desc);
+ return rc;
+}
+
static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
{
int i;
diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 1f17537..d93cf51 100644
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -557,6 +557,7 @@ void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid,
void smc_smcd_terminate_all(struct smcd_dev *dev);
void smc_smcr_terminate_all(struct smc_ib_device *smcibdev);
int smc_buf_create(struct smc_sock *smc, bool is_smcd);
+int smcd_buf_attach(struct smc_sock *smc);
int smc_uncompress_bufsize(u8 compressed);
int smc_rmb_rtoken_handling(struct smc_connection *conn, struct smc_link *link,
struct smc_clc_msg_accept_confirm *clc);
--
1.8.3.1
In some scenarios using virtual ISM device, sndbuf can share the same
physical memory region with peer DMB to avoid data copy from one side
to the other. In such case the sndbuf is only a descriptor that
describes the shared memory and does not actually occupy memory, it's
more like a ghost buffer.
+----------+ +----------+
| socket A | | socket B |
+----------+ +----------+
| |
+--------+ +--------+
| sndbuf | | DMB |
| desc | | desc |
+--------+ +--------+
| |
| +----v-----+
+--------------------------> memory |
+----------+
So here introduces two new SMC-D device operations for attaching or
detaching ghost sndbuf to peer DMB. For now only intra-VM communication
with loopback-ism device supports this.
Signed-off-by: Wen Gu <[email protected]>
---
include/net/smc.h | 2 ++
net/smc/smc_ism.c | 38 ++++++++++++++++++++++++++++++++++++++
net/smc/smc_ism.h | 4 ++++
3 files changed, 44 insertions(+)
diff --git a/include/net/smc.h b/include/net/smc.h
index 6273c3a..648c2f8 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -62,6 +62,8 @@ struct smcd_ops {
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
void *client);
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
+ int (*attach_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
+ int (*detach_dmb)(struct smcd_dev *dev, u64 token);
int (*add_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*del_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*set_vlan_required)(struct smcd_dev *dev);
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 4065ebd..24a51ed 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -246,6 +246,44 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
return rc;
}
+bool smc_ism_dmb_mappable(struct smcd_dev *smcd)
+{
+ /* for now only loopback-ism supports mapping
+ * ghost sndbuf to peer DMB.
+ */
+ return (smcd->ops->get_chid(smcd) == 0xFFFF);
+}
+
+int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
+ struct smc_buf_desc *dmb_desc)
+{
+ struct smcd_dmb dmb;
+ int rc = 0;
+
+ if (!dev->ops->attach_dmb)
+ return -EINVAL;
+
+ memset(&dmb, 0, sizeof(dmb));
+ dmb.dmb_tok = token;
+ rc = dev->ops->attach_dmb(dev, &dmb);
+ if (!rc) {
+ dmb_desc->sba_idx = dmb.sba_idx;
+ dmb_desc->token = dmb.dmb_tok;
+ dmb_desc->cpu_addr = dmb.cpu_addr;
+ dmb_desc->dma_addr = dmb.dma_addr;
+ dmb_desc->len = dmb.dmb_len;
+ }
+ return rc;
+}
+
+int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token)
+{
+ if (!dev->ops->detach_dmb)
+ return -EINVAL;
+
+ return dev->ops->detach_dmb(dev, token);
+}
+
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
struct sk_buff *skb,
struct netlink_callback *cb)
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index 6903cd5..8ed1f76 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -48,6 +48,10 @@ int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id,
int smc_ism_register_dmb(struct smc_link_group *lgr, int buf_size,
struct smc_buf_desc *dmb_desc);
int smc_ism_unregister_dmb(struct smcd_dev *dev, struct smc_buf_desc *dmb_desc);
+bool smc_ism_dmb_mappable(struct smcd_dev *smcd);
+int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
+ struct smc_buf_desc *dmb_desc);
+int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token);
int smc_ism_signal_shutdown(struct smc_link_group *lgr);
void smc_ism_get_system_eid(u8 **eid);
u16 smc_ism_get_chid(struct smcd_dev *dev);
--
1.8.3.1
Since ghost sndbuf shares the same physical memory with peer DMB,
the cursor update processing needs to be adapted to ensure that the
data to be consumed won't be overwritten.
So in this case, the fin_curs and sndbuf_space that were originally
updated after sending the CDC message should be modified to not be
update until the peer updates cons_curs.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_cdc.c | 52 +++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 41 insertions(+), 11 deletions(-)
diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c
index c820ef1..f2b0bcc 100644
--- a/net/smc/smc_cdc.c
+++ b/net/smc/smc_cdc.c
@@ -18,6 +18,7 @@
#include "smc_tx.h"
#include "smc_rx.h"
#include "smc_close.h"
+#include "smc_ism.h"
/********************************** send *************************************/
@@ -255,17 +256,25 @@ int smcd_cdc_msg_send(struct smc_connection *conn)
return rc;
smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn);
conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;
- /* Calculate transmitted data and increment free send buffer space */
- diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,
- &conn->tx_curs_sent);
- /* increased by confirmed number of bytes */
- smp_mb__before_atomic();
- atomic_add(diff, &conn->sndbuf_space);
- /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
- smp_mb__after_atomic();
- smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);
+ if (!smc_ism_dmb_mappable(conn->lgr->smcd)) {
+ /* Ghost sndbuf shares the same memory region with
+ * peer DMB, so don't update the tx_curs_fin and
+ * sndbuf_space until peer has consumed the data.
+ */
+ /* Calculate transmitted data and increment free
+ * send buffer space
+ */
+ diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,
+ &conn->tx_curs_sent);
+ /* increased by confirmed number of bytes */
+ smp_mb__before_atomic();
+ atomic_add(diff, &conn->sndbuf_space);
+ /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
+ smp_mb__after_atomic();
+ smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);
- smc_tx_sndbuf_nonfull(smc);
+ smc_tx_sndbuf_nonfull(smc);
+ }
return rc;
}
@@ -323,7 +332,7 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc,
{
union smc_host_cursor cons_old, prod_old;
struct smc_connection *conn = &smc->conn;
- int diff_cons, diff_prod;
+ int diff_cons, diff_prod, diff_tx;
smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn);
smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn);
@@ -339,6 +348,27 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc,
atomic_add(diff_cons, &conn->peer_rmbe_space);
/* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */
smp_mb__after_atomic();
+
+ if (conn->lgr->is_smcd &&
+ smc_ism_dmb_mappable(conn->lgr->smcd)) {
+ /* Ghost sndbuf shares the same memory region with
+ * peer RMB, so update tx_curs_fin and sndbuf_space
+ * when peer has consumed the data.
+ */
+ /* calculate peer rmb consumed data */
+ diff_tx = smc_curs_diff(conn->sndbuf_desc->len,
+ &conn->tx_curs_fin,
+ &conn->local_rx_ctrl.cons);
+ /* increase local sndbuf space and fin_curs */
+ smp_mb__before_atomic();
+ atomic_add(diff_tx, &conn->sndbuf_space);
+ /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
+ smp_mb__after_atomic();
+ smc_curs_copy(&conn->tx_curs_fin,
+ &conn->local_rx_ctrl.cons, conn);
+
+ smc_tx_sndbuf_nonfull(smc);
+ }
}
diff_prod = smc_curs_diff(conn->rmb_desc->len, &prod_old,
--
1.8.3.1
This implements attach_dmb and detach_dmb interfaces of loopback-ism.
The DMB won't be unregistered by loopback-ism until no sndbuf is
attached to it.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 107 +++++++++++++++++++++++++++++++++++++++++++------
net/smc/smc_loopback.h | 4 ++
2 files changed, 99 insertions(+), 12 deletions(-)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 057ea6a..909c069 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -198,6 +198,7 @@ static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
}
dmb_node->len = dmb->dmb_len;
dmb_node->dma_addr = SMC_DMA_ADDR_INVALID;
+ refcount_set(&dmb_node->refcnt, 1);
again:
/* add new dmb into hash table */
@@ -211,6 +212,7 @@ static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
}
hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
write_unlock(&ldev->dmb_ht_lock);
+ atomic_inc(&ldev->dmb_cnt);
SMC_LO_STAT_DMBS_INC(ldev);
dmb->sba_idx = dmb_node->sba_idx;
@@ -233,8 +235,8 @@ static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
struct smc_lo_dev *ldev = smcd->priv;
- /* remove dmb from hash table */
- write_lock(&ldev->dmb_ht_lock);
+ /* find dmb from hash table */
+ read_lock(&ldev->dmb_ht_lock);
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
if (tmp_node->token == dmb->dmb_tok) {
dmb_node = tmp_node;
@@ -242,9 +244,18 @@ static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
}
}
if (!dmb_node) {
- write_unlock(&ldev->dmb_ht_lock);
+ read_unlock(&ldev->dmb_ht_lock);
return -EINVAL;
}
+ read_unlock(&ldev->dmb_ht_lock);
+
+ /* wait for peer sndbuf to detach from this dmb */
+ if (!refcount_dec_and_test(&dmb_node->refcnt))
+ wait_event(ldev->dmbs_release,
+ !refcount_read(&dmb_node->refcnt));
+
+ /* remove dmb from hash table */
+ write_lock(&ldev->dmb_ht_lock);
hash_del(&dmb_node->list);
write_unlock(&ldev->dmb_ht_lock);
@@ -253,6 +264,66 @@ static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
kfree(dmb_node);
SMC_LO_STAT_DMBS_DEC(ldev);
+ if (atomic_dec_and_test(&ldev->dmb_cnt))
+ wake_up(&ldev->ldev_release);
+ return 0;
+}
+
+static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
+{
+ struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* find dmb_node according to dmb->dmb_tok */
+ read_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
+ if (tmp_node->token == dmb->dmb_tok) {
+ dmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!dmb_node) {
+ read_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ read_unlock(&ldev->dmb_ht_lock);
+
+ if (!refcount_inc_not_zero(&dmb_node->refcnt))
+ /* the dmb is being unregistered, but has
+ * not been removed from the hash table.
+ */
+ return -EINVAL;
+
+ /* provide dmb information */
+ dmb->sba_idx = dmb_node->sba_idx;
+ dmb->dmb_tok = dmb_node->token;
+ dmb->cpu_addr = dmb_node->cpu_addr;
+ dmb->dma_addr = dmb_node->dma_addr;
+ dmb->dmb_len = dmb_node->len;
+ return 0;
+}
+
+static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token)
+{
+ struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* find dmb_node according to dmb->dmb_tok */
+ read_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
+ if (tmp_node->token == token) {
+ dmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!dmb_node) {
+ read_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ read_unlock(&ldev->dmb_ht_lock);
+
+ if (refcount_dec_and_test(&dmb_node->refcnt))
+ wake_up_all(&ldev->dmbs_release);
return 0;
}
@@ -288,7 +359,15 @@ static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok,
{
struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node;
struct smc_lo_dev *ldev = smcd->priv;
-
+ struct smc_connection *conn;
+
+ if (!sf) {
+ /* Since ghost sndbuf shares the same memory region
+ * with peer DMB, there is no need to 'transfer' data
+ * from sndbuf to peer DMB.
+ */
+ return 0;
+ }
read_lock(&ldev->dmb_ht_lock);
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
if (tmp_node->token == dmb_tok) {
@@ -305,14 +384,10 @@ static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok,
memcpy((char *)rmb_node->cpu_addr + offset, data, size);
SMC_LO_STAT_TX_BYTES(ldev, size);
- if (sf) {
- struct smc_connection *conn =
- smcd->conn[rmb_node->sba_idx];
-
- if (conn && !conn->killed) {
- SMC_LO_STAT_RX_BYTES(ldev, size);
- smcd_cdc_rx_handler(conn);
- }
+ conn = smcd->conn[rmb_node->sba_idx];
+ if (conn && !conn->killed) {
+ SMC_LO_STAT_RX_BYTES(ldev, size);
+ smcd_cdc_rx_handler(conn);
}
return 0;
}
@@ -345,6 +420,8 @@ static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
.query_remote_gid = smc_lo_query_rgid,
.register_dmb = smc_lo_register_dmb,
.unregister_dmb = smc_lo_unregister_dmb,
+ .attach_dmb = smc_lo_attach_dmb,
+ .detach_dmb = smc_lo_detach_dmb,
.add_vlan_id = smc_lo_add_vlan_id,
.del_vlan_id = smc_lo_del_vlan_id,
.set_vlan_required = smc_lo_set_vlan_required,
@@ -426,12 +503,18 @@ static int smc_lo_dev_init(struct smc_lo_dev *ldev)
smc_lo_generate_id(ldev);
rwlock_init(&ldev->dmb_ht_lock);
hash_init(ldev->dmb_ht);
+ atomic_set(&ldev->dmb_cnt, 0);
+ init_waitqueue_head(&ldev->dmbs_release);
+ init_waitqueue_head(&ldev->ldev_release);
+
return smcd_lo_register_dev(ldev);
}
static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
{
smcd_lo_unregister_dev(ldev);
+ if (atomic_read(&ldev->dmb_cnt))
+ wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
}
static void smc_lo_dev_release(struct device *dev)
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index ad0feaa..df6c252 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -30,6 +30,7 @@ struct smc_lo_dmb_node {
u32 sba_idx;
void *cpu_addr;
dma_addr_t dma_addr;
+ refcount_t refcnt;
};
struct smc_lo_dev_stats64 {
@@ -45,9 +46,12 @@ struct smc_lo_dev {
u16 chid;
struct smcd_gid local_gid;
struct smc_lo_dev_stats64 __percpu *stats;
+ atomic_t dmb_cnt;
rwlock_t dmb_ht_lock;
DECLARE_BITMAP(sba_idx_mask, SMC_LO_MAX_DMBS);
DECLARE_HASHTABLE(dmb_ht, SMC_LO_DMBS_HASH_BITS);
+ wait_queue_head_t dmbs_release;
+ wait_queue_head_t ldev_release;
};
#define SMC_LO_STAT_SUB(ldev, key, val) \
--
1.8.3.1