Hi, all
# Background
SMC-D is now used in IBM z with ISM function to optimize network interconnect
for intra-CPC communications. Inspired by this, we try to make SMC-D available
on the non-s390 architecture through a software-simulated virtual ISM device,
such as loopback-ism device here, to accelerate inter-process or inter-containers
communication within the same OS.
# Design
This patch set includes 4 parts:
- Patch #1-#3: decouple ISM device hard code from SMC-D stack.
- Patch #4-#8: implement virtual ISM extension defined in SMCv2.1.
- Patch #9-#13: implement loopback-ism device.
- Patch #14-#18: memory copy optimization for the case using loopback.
The loopback-ism device is designed as a kernel device and not be limited to
a specific net namespace, ends of both inter-process connection (1/1' in diagram
below) or inter-container connection (2/2' in diagram below) will find that peer
shares the same loopback-ism device during the CLC handshake. Then loopback-ism
device will be chosen.
Container 1 (ns1) Container 2 (ns2)
+-----------------------------------------+ +-------------------------+
| +-------+ +-------+ +-------+ | | +-------+ |
| | App A | | App B | | App C | | | | App D |<-+ |
| +-------+ +---^---+ +-------+ | | +-------+ |(2') |
| |127.0.0.1 (1')| |192.168.0.11 192.168.0.12| |
| (1)| +--------+ | +--------+ |(2) | | +--------+ +--------+ |
| `-->| lo |-` | eth0 |<-` | | | lo | | eth0 | |
+---------+--|---^-+---+-----|--+---------+ +-+--------+---+-^------+-+
| | | |
Kernel | | | |
+----+-------v---+-----------v----------------------------------+---+----+
| | TCP | |
| | | |
| +--------------------------------------------------------------+ |
| |
| +--------------+ |
| | smc loopback | |
+---------------------------+--------------+-----------------------------+
loopback-ism device allocs RMBs and sndbufs for each connection peer and 'moves'
data from sndbuf at one end to RMB at the other end. Since communication occurs
within the same kernel, the sndbuf can be mapped to peer RMB so that the data
copy in loopback-ism case can be avoided.
Container 1 (ns1) Container 2 (ns2)
+-----------------------------------------+ +-------------------------+
| +-------+ +-------+ +-------+ | | +-------+ |
| | App A | | App B | | App C | | | | App D | |
| +-------+ +--^----+ +-------+ | | +---^---+ |
| | | | | | | |
| (1) | (1') | (2) | | | (2') | |
| | | | | | | |
+-------|-----------|---------------|-----+ +------------|------------+
| | | |
Kernel | | | |
+-------|-----------|---------------|-----------------------|------------+
| +-----v-+ +-------+ +---v---+ +-------+ |
| | snd A |-+ | RMB B |<--+ | snd C |-+ +->| RMB D | |
| +-------+ | +-------+ | +-------+ | | +-------+ |
| +-------+ | +-------+ | +-------+ | | +-------+ |
| | RMB A | | | snd B | | | RMB C | | | | snd D | |
| +-------+ | +-------+ | +-------+ | | +-------+ |
| | +-------------v+ | |
| +-------------->| smc loopback |---------+ |
+---------------------------+--------------+-----------------------------+
# Benchmark Test
* Test environments:
- VM with Intel Xeon Platinum 8 core 2.50GHz, 16 GiB mem.
- SMC sndbuf/RMB size 1MB.
* Test object:
- TCP: run on TCP loopback.
- domain: run on UNIX domain.
- SMC lo: run on SMC loopback device.
1. ipc-benchmark (see [1])
- ./<foo> -c 1000000 -s 100
TCP domain SMC-lo
Message
rate (msg/s) 78855 107621(+36.41%) 153351(+94.47%)
2. sockperf
- serv: <smc_run> taskset -c <cpu> sockperf sr --tcp
- clnt: <smc_run> taskset -c <cpu> sockperf { tp | pp } --tcp --msg-size={ 64000 for tp | 14 for pp } -i 127.0.0.1 -t 30
TCP SMC-lo
Bandwidth(MBps) 5169.250 8007.080(+54.89%)
Latency(us) 6.122 3.174(-48.15%)
3. nginx/wrk
- serv: <smc_run> nginx
- clnt: <smc_run> wrk -t 8 -c 1000 -d 30 http://127.0.0.1:80
TCP SMC-lo
Requests/s 197432.19 261056.09(+32.22%)
4. redis-benchmark
- serv: <smc_run> redis-server
- clnt: <smc_run> redis-benchmark -h 127.0.0.1 -q -t set,get -n 400000 -c 200 -d 1024
TCP SMC-lo
GET(Requests/s) 86244.07 122025.62(+41.48%)
SET(Requests/s) 86749.08 120048.02(+38.38%)
[1] https://github.com/goldsborough/ipc-bench
Wen Gu (18):
net/smc: decouple ism_dev from SMC-D device dump
net/smc: decouple ism_dev from SMC-D DMB registration
net/smc: extract v2 check helper from SMC-D device registration
net/smc: support SMCv2.x supplemental features negotiation
net/smc: reserve CHID range for SMC-D virtual device
net/smc: extend GID to 128bits for virtual ISM device
net/smc: disable SEID on non-s390 architecture
net/smc: enable virtual ISM device feature bit
net/smc: introduce SMC-D loopback device
net/smc: implement ID-related operations of loopback
net/smc: implement some unsupported operations of loopback
net/smc: implement DMB-related operations of loopback
net/smc: register loopback device as SMC-Dv2 device
net/smc: add operation for getting DMB attribute
net/smc: add operations for DMB attach and detach
net/smc: avoid data copy from sndbuf to peer RMB in SMC-D
net/smc: modify cursor update logic when sndbuf mapped to RMB
net/smc: add interface implementation of loopback device
drivers/s390/net/ism_drv.c | 13 +-
include/net/smc.h | 28 ++-
include/uapi/linux/smc.h | 3 +
net/smc/Makefile | 2 +-
net/smc/af_smc.c | 73 +++++--
net/smc/smc.h | 7 +
net/smc/smc_cdc.c | 56 ++++--
net/smc/smc_cdc.h | 1 +
net/smc/smc_clc.c | 56 ++++--
net/smc/smc_clc.h | 10 +-
net/smc/smc_core.c | 108 +++++++++-
net/smc/smc_core.h | 9 +-
net/smc/smc_diag.c | 6 +-
net/smc/smc_ism.c | 96 ++++++---
net/smc/smc_ism.h | 24 ++-
net/smc/smc_loopback.c | 482 +++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_loopback.h | 52 +++++
net/smc/smc_pnet.c | 4 +-
18 files changed, 938 insertions(+), 92 deletions(-)
create mode 100644 net/smc/smc_loopback.c
create mode 100644 net/smc/smc_loopback.h
--
1.8.3.1
This patch introduces a kind of loopback device for SMC-D. The device
is created when SMC module is loaded and destroyed when the SMC module
is unloaded. The loopback device is a kernel device used only by the
SMC module and is not restricted by net namespace, so it can be used
for local inter-process or inter-container communication.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/Makefile | 2 +-
net/smc/af_smc.c | 12 +++-
net/smc/smc_loopback.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++
net/smc/smc_loopback.h | 31 ++++++++++
4 files changed, 200 insertions(+), 2 deletions(-)
create mode 100644 net/smc/smc_loopback.c
create mode 100644 net/smc/smc_loopback.h
diff --git a/net/smc/Makefile b/net/smc/Makefile
index 875efcd..a8c3711 100644
--- a/net/smc/Makefile
+++ b/net/smc/Makefile
@@ -4,5 +4,5 @@ obj-$(CONFIG_SMC) += smc.o
obj-$(CONFIG_SMC_DIAG) += smc_diag.o
smc-y := af_smc.o smc_pnet.o smc_ib.o smc_clc.o smc_core.o smc_wr.o smc_llc.o
smc-y += smc_cdc.o smc_tx.o smc_rx.o smc_close.o smc_ism.o smc_netlink.o smc_stats.o
-smc-y += smc_tracepoint.o
+smc-y += smc_tracepoint.o smc_loopback.o
smc-$(CONFIG_SYSCTL) += smc_sysctl.o
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 7eab600..bc4300e 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -53,6 +53,7 @@
#include "smc_stats.h"
#include "smc_tracepoint.h"
#include "smc_sysctl.h"
+#include "smc_loopback.h"
static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
* creation on server
@@ -3552,15 +3553,23 @@ static int __init smc_init(void)
goto out_sock;
}
+ rc = smc_loopback_init();
+ if (rc) {
+ pr_err("%s: smc_loopback_init fails with %d\n", __func__, rc);
+ goto out_ib;
+ }
+
rc = tcp_register_ulp(&smc_ulp_ops);
if (rc) {
pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
- goto out_ib;
+ goto out_lo;
}
static_branch_enable(&tcp_have_smc);
return 0;
+out_lo:
+ smc_loopback_exit();
out_ib:
smc_ib_unregister_client();
out_sock:
@@ -3598,6 +3607,7 @@ static void __exit smc_exit(void)
tcp_unregister_ulp(&smc_ulp_ops);
sock_unregister(PF_SMC);
smc_core_exit();
+ smc_loopback_exit();
smc_ib_unregister_client();
smc_ism_exit();
destroy_workqueue(smc_close_wq);
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
new file mode 100644
index 0000000..7d88856
--- /dev/null
+++ b/net/smc/smc_loopback.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shared Memory Communications Direct over loopback device.
+ *
+ * Provide a SMC-D loopback dummy device.
+ *
+ * Copyright (c) 2022, Alibaba Inc.
+ *
+ * Author: Wen Gu <[email protected]>
+ * Tony Lu <[email protected]>
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/types.h>
+#include <net/smc.h>
+
+#include "smc_ism.h"
+#include "smc_loopback.h"
+
+static const char smc_lo_dev_name[] = "smc_lo";
+static struct smc_lo_dev *lo_dev;
+
+static const struct smcd_ops lo_ops = {
+ .query_remote_gid = NULL,
+ .register_dmb = NULL,
+ .unregister_dmb = NULL,
+ .add_vlan_id = NULL,
+ .del_vlan_id = NULL,
+ .set_vlan_required = NULL,
+ .reset_vlan_required = NULL,
+ .signal_event = NULL,
+ .move_data = NULL,
+ .supports_v2 = NULL,
+ .get_system_eid = NULL,
+ .get_local_gid = NULL,
+ .get_chid = NULL,
+ .get_dev = NULL,
+};
+
+static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops,
+ int max_dmbs)
+{
+ struct smcd_dev *smcd;
+
+ smcd = kzalloc(sizeof(*smcd), GFP_KERNEL);
+ if (!smcd)
+ return NULL;
+
+ smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *),
+ GFP_KERNEL);
+ if (!smcd->conn)
+ goto out_smcd;
+
+ smcd->ops = ops;
+
+ spin_lock_init(&smcd->lock);
+ spin_lock_init(&smcd->lgr_lock);
+ INIT_LIST_HEAD(&smcd->vlan);
+ INIT_LIST_HEAD(&smcd->lgr_list);
+ init_waitqueue_head(&smcd->lgrs_deleted);
+ return smcd;
+
+out_smcd:
+ kfree(smcd);
+ return NULL;
+}
+
+static int smcd_lo_register_dev(struct smc_lo_dev *ldev)
+{
+ struct smcd_dev *smcd;
+
+ smcd = smcd_lo_alloc_dev(&lo_ops, SMC_LODEV_MAX_DMBS);
+ if (!smcd)
+ return -ENOMEM;
+
+ ldev->smcd = smcd;
+ smcd->priv = ldev;
+
+ /* TODO:
+ * register smc_lo to smcd_dev list.
+ */
+ return 0;
+}
+
+static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
+{
+ /* TODO:
+ * unregister smc_lo from smcd_dev list.
+ */
+}
+
+static void smc_lo_dev_release(struct device *dev)
+{
+ struct smc_lo_dev *ldev =
+ container_of(dev, struct smc_lo_dev, dev);
+ struct smcd_dev *smcd = ldev->smcd;
+
+ kfree(smcd->conn);
+ kfree(smcd);
+ kfree(ldev);
+}
+
+static int smc_lo_dev_init(struct smc_lo_dev *ldev)
+{
+ return smcd_lo_register_dev(ldev);
+}
+
+static int smc_lo_dev_probe(void)
+{
+ struct smc_lo_dev *ldev;
+ int ret;
+
+ ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
+ if (!ldev)
+ return -ENOMEM;
+
+ ldev->dev.parent = NULL;
+ ldev->dev.release = smc_lo_dev_release;
+ device_initialize(&ldev->dev);
+ dev_set_name(&ldev->dev, smc_lo_dev_name);
+
+ ret = smc_lo_dev_init(ldev);
+ if (ret)
+ goto free_dev;
+
+ lo_dev = ldev; /* global loopback device */
+ return 0;
+
+free_dev:
+ kfree(ldev);
+ return ret;
+}
+
+static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
+{
+ smcd_lo_unregister_dev(ldev);
+}
+
+static void smc_lo_dev_remove(void)
+{
+ if (!lo_dev)
+ return;
+
+ smc_lo_dev_exit(lo_dev);
+ put_device(&lo_dev->dev); /* device_initialize in smc_lo_dev_probe */
+}
+
+int smc_loopback_init(void)
+{
+ return smc_lo_dev_probe();
+}
+
+void smc_loopback_exit(void)
+{
+ smc_lo_dev_remove();
+}
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
new file mode 100644
index 0000000..0f7583c
--- /dev/null
+++ b/net/smc/smc_loopback.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shared Memory Communications Direct over loopback device.
+ *
+ * Provide a SMC-D loopback dummy device.
+ *
+ * Copyright (c) 2022, Alibaba Inc.
+ *
+ * Author: Wen Gu <[email protected]>
+ * Tony Lu <[email protected]>
+ *
+ */
+
+#ifndef _SMC_LOOPBACK_H
+#define _SMC_LOOPBACK_H
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <net/smc.h>
+
+#define SMC_LODEV_MAX_DMBS 5000
+
+struct smc_lo_dev {
+ struct smcd_dev *smcd;
+ struct device dev;
+};
+
+int smc_loopback_init(void);
+void smc_loopback_exit(void);
+
+#endif /* _SMC_LOOPBACK_H */
--
1.8.3.1
This patch aims to avoid data copy from local sndbuf to peer RMB by mapping
local sndbuf to peer RMB when DMBs have ISM_ATTR_DMB_MAP attribute.
After this, local sndbuf and peer RMB share the same physical memory.
+----------+ +----------+
| socket A | | socket B |
+----------+ +----------+
| ^
| +---------+ |
regard as | | regard as
local sndbuf | B's | local RMB
| | RMB | |
|-------> | |-----------|
+---------+
1. From the perspective of RMB:
a. Created or reused when connection is created.
b. Unused and recycled to lgr buffer pool when connection is freed.
c. Freed when link group is freed.
2. From the perspective of sndbuf:
a. Mapped to peer RMB by the rtoken exchanged through CLC message.
Then accessing local sndbuf is equivalent to accessing peer RMB.
c. Unmapped from peer RMB and freed when connection is freed. Won't be
recycled to lgr buffer pool.
Therefore, the data written to local sndbuf will directly reach peer RMB.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/af_smc.c | 14 +++++++++++
net/smc/smc_core.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
net/smc/smc_core.h | 1 +
3 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index bc4300e..fd0b91f 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -1436,6 +1436,12 @@ static int smc_connect_ism(struct smc_sock *smc,
}
smc_conn_save_peer_info(smc, aclc);
+
+ if (smc_ism_dmb_mappable(smc->conn.lgr->smcd)) {
+ rc = smcd_buf_attach(smc);
+ if (rc)
+ goto connect_abort;
+ }
smc_close_init(smc);
smc_rx_init(smc);
smc_tx_init(smc);
@@ -2537,6 +2543,14 @@ static void smc_listen_work(struct work_struct *work)
mutex_unlock(&smc_server_lgr_pending);
}
smc_conn_save_peer_info(new_smc, cclc);
+
+ if (ini->is_smcd &&
+ smc_ism_dmb_mappable(new_smc->conn.lgr->smcd)) {
+ rc = smcd_buf_attach(new_smc);
+ if (rc)
+ goto out_decl;
+ }
+
smc_listen_out_connected(new_smc);
SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
goto out_free;
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index c36500a..bae2116 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -1153,6 +1153,20 @@ static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb,
}
}
+static void smcd_buf_detach(struct smc_connection *conn)
+{
+ struct smcd_dev *smcd = conn->lgr->smcd;
+ u64 peer_token = conn->peer_token;
+
+ if (!conn->sndbuf_desc)
+ return;
+
+ smc_ism_detach_dmb(smcd, peer_token);
+
+ kfree(conn->sndbuf_desc);
+ conn->sndbuf_desc = NULL;
+}
+
static void smc_buf_unuse(struct smc_connection *conn,
struct smc_link_group *lgr)
{
@@ -1197,6 +1211,10 @@ void smc_conn_free(struct smc_connection *conn)
if (!list_empty(&lgr->list))
smc_ism_unset_conn(conn);
tasklet_kill(&conn->rx_tsklet);
+
+ /* detach sndbuf from peer RMB */
+ if (smc_ism_dmb_mappable(lgr->smcd))
+ smcd_buf_detach(conn);
} else {
smc_cdc_wait_pend_tx_wr(conn);
if (current_work() != &conn->abort_work)
@@ -2458,15 +2476,23 @@ void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
*/
int smc_buf_create(struct smc_sock *smc, bool is_smcd)
{
+ bool sndbuf_created = false;
int rc;
+ if (is_smcd &&
+ smc_ism_dmb_mappable(smc->conn.lgr->smcd))
+ goto create_rmb;
+
/* create send buffer */
rc = __smc_buf_create(smc, is_smcd, false);
if (rc)
return rc;
+ sndbuf_created = true;
+
+create_rmb:
/* create rmb */
rc = __smc_buf_create(smc, is_smcd, true);
- if (rc) {
+ if (rc && sndbuf_created) {
down_write(&smc->conn.lgr->sndbufs_lock);
list_del(&smc->conn.sndbuf_desc->list);
up_write(&smc->conn.lgr->sndbufs_lock);
@@ -2476,6 +2502,48 @@ int smc_buf_create(struct smc_sock *smc, bool is_smcd)
return rc;
}
+int smcd_buf_attach(struct smc_sock *smc)
+{
+ struct smc_connection *conn = &smc->conn;
+ struct smcd_dev *smcd = conn->lgr->smcd;
+ u64 peer_token = conn->peer_token;
+ struct smc_buf_desc *buf_desc;
+ int rc;
+
+ buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
+ if (!buf_desc)
+ return -ENOMEM;
+
+ /* map local sndbuf desc to peer RMB, so operations on local
+ * sndbuf are equivalent to operations on peer RMB.
+ */
+ rc = smc_ism_attach_dmb(smcd, peer_token, buf_desc);
+ if (rc) {
+ rc = SMC_CLC_DECL_MEM;
+ goto free;
+ }
+
+ smc->sk.sk_sndbuf = buf_desc->len;
+ buf_desc->cpu_addr = (u8 *)buf_desc->cpu_addr + sizeof(struct smcd_cdc_msg);
+ buf_desc->len -= sizeof(struct smcd_cdc_msg);
+ conn->sndbuf_desc = buf_desc;
+ conn->sndbuf_desc->used = 1;
+ atomic_set(&conn->sndbuf_space, conn->sndbuf_desc->len);
+ return 0;
+
+free:
+ if (conn->rmb_desc) {
+ /* free local RMB as well */
+ down_write(&conn->lgr->rmbs_lock);
+ list_del(&conn->rmb_desc->list);
+ up_write(&conn->lgr->rmbs_lock);
+ smc_buf_free(conn->lgr, true, conn->rmb_desc);
+ conn->rmb_desc = NULL;
+ }
+ kfree(buf_desc);
+ return rc;
+}
+
static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
{
int i;
diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index d57eb9b..2cba119 100644
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -551,6 +551,7 @@ void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid,
void smc_smcd_terminate_all(struct smcd_dev *dev);
void smc_smcr_terminate_all(struct smc_ib_device *smcibdev);
int smc_buf_create(struct smc_sock *smc, bool is_smcd);
+int smcd_buf_attach(struct smc_sock *smc);
int smc_uncompress_bufsize(u8 compressed);
int smc_rmb_rtoken_handling(struct smc_connection *conn, struct smc_link *link,
struct smc_clc_msg_accept_confirm *clc);
--
1.8.3.1
This patch extends smcd_ops, adding two more semantic for SMC-D DMB:
- attach_dmb:
Attach an already registered DMB to a specific buf_desc, so that we
can refer to the DMB through this buf_desc.
- detach_dmb:
Reverse operation of attach_dmb. detach the DMB from the buf_desc.
This interface extension is to prepare for the avoidance of memory copy
from sndbuf to RMB with SMC-D device whose DMBs has ISM_ATTR_DMB_MAP
attribute.
Signed-off-by: Wen Gu <[email protected]>
---
include/net/smc.h | 2 ++
net/smc/smc_ism.c | 31 +++++++++++++++++++++++++++++++
net/smc/smc_ism.h | 2 ++
3 files changed, 35 insertions(+)
diff --git a/include/net/smc.h b/include/net/smc.h
index 6930c8a..04dbaa4 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -72,6 +72,8 @@ struct smcd_ops {
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
void *client);
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
+ int (*attach_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
+ int (*detach_dmb)(struct smcd_dev *dev, u64 token);
int (*add_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*del_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*set_vlan_required)(struct smcd_dev *dev);
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 14e018c..0c20745 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -244,6 +244,37 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
return rc;
}
+int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
+ struct smc_buf_desc *dmb_desc)
+{
+ struct smcd_dmb dmb;
+ int rc = 0;
+
+ memset(&dmb, 0, sizeof(dmb));
+ dmb.dmb_tok = token;
+
+ if (!dev->ops->attach_dmb)
+ return -EINVAL;
+
+ rc = dev->ops->attach_dmb(dev, &dmb);
+ if (!rc) {
+ dmb_desc->sba_idx = dmb.sba_idx;
+ dmb_desc->token = dmb.dmb_tok;
+ dmb_desc->cpu_addr = dmb.cpu_addr;
+ dmb_desc->dma_addr = dmb.dma_addr;
+ dmb_desc->len = dmb.dmb_len;
+ }
+ return rc;
+}
+
+int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token)
+{
+ if (!dev->ops->detach_dmb)
+ return -EINVAL;
+
+ return dev->ops->detach_dmb(dev, token);
+}
+
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
struct sk_buff *skb,
struct netlink_callback *cb)
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index cef212c..f30aae4 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -45,6 +45,8 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int buf_size,
struct smc_buf_desc *dmb_desc);
int smc_ism_unregister_dmb(struct smcd_dev *dev, struct smc_buf_desc *dmb_desc);
bool smc_ism_dmb_mappable(struct smcd_dev *smcd);
+int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token, struct smc_buf_desc *dmb_desc);
+int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token);
int smc_ism_signal_shutdown(struct smc_link_group *lgr);
void smc_ism_get_system_eid(u8 **eid);
u16 smc_ism_get_chid(struct smcd_dev *dev);
--
1.8.3.1
This patch adds SMCv2.1 supplemental features negotiation. Supported
SMCv2.1 supplemental features are represented by feature_mask in FCE
header of CLC messages.
Server Client
Proposal(features(c-mask bits))
<-----------------------------------
Accept(features(s-mask bits))
----------------------------------->
Confirm(features(s&c-mask bits))
<-----------------------------------
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_clc.c | 7 +++++++
net/smc/smc_clc.h | 6 ++++--
net/smc/smc_core.h | 1 +
3 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c
index 8deb46c..125b0d2 100644
--- a/net/smc/smc_clc.c
+++ b/net/smc/smc_clc.c
@@ -426,6 +426,7 @@ static int smc_clc_fill_fce(struct smc_clc_first_contact_ext_v2x *fce,
memset(fce, 0, sizeof(*fce));
fce->fce_v2_base.os_type = SMC_CLC_OS_LINUX;
fce->fce_v2_base.release = ini->release_nr;
+ fce->fce_v2_base.feature_mask = htons(ini->feature_mask);
memcpy(fce->fce_v2_base.hostname, smc_hostname, sizeof(smc_hostname));
if (ini->is_smcd && ini->release_nr < SMC_RELEASE_1) {
ret = sizeof(struct smc_clc_first_contact_ext);
@@ -906,6 +907,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
pclc_smcd->v2_ext_offset = htons(v2_ext_offset);
plen += sizeof(*v2_ext);
+ v2_ext->feature_mask = htons(ini->feature_mask);
read_lock(&smc_clc_eid_table.lock);
v2_ext->hdr.eid_cnt = smc_clc_eid_table.ueid_cnt;
plen += smc_clc_eid_table.ueid_cnt * SMC_MAX_EID_LEN;
@@ -1219,6 +1221,7 @@ int smc_clc_clnt_v2x_features_validate(struct smc_clc_first_contact_ext *fce,
return SMC_CLC_DECL_MAXLINKERR;
ini->max_links = fce_v2x->max_links;
}
+ ini->feature_mask &= ntohs(fce->feature_mask);
return 0;
}
@@ -1250,6 +1253,10 @@ int smc_clc_v2x_features_confirm_check(struct smc_clc_msg_accept_confirm *cclc,
return SMC_CLC_DECL_MAXLINKERR;
}
+ if (~(ini->feature_mask) & ntohs(fce->feature_mask))
+ return SMC_CLC_DECL_FEATUNSUPP;
+ ini->feature_mask = ntohs(fce->feature_mask);
+
return 0;
}
diff --git a/net/smc/smc_clc.h b/net/smc/smc_clc.h
index c5c8e7d..bcf37c8 100644
--- a/net/smc/smc_clc.h
+++ b/net/smc/smc_clc.h
@@ -48,6 +48,7 @@
#define SMC_CLC_DECL_RELEASEERR 0x03030009 /* release version negotiate failed */
#define SMC_CLC_DECL_MAXCONNERR 0x0303000a /* max connections negotiate failed */
#define SMC_CLC_DECL_MAXLINKERR 0x0303000b /* max links negotiate failed */
+#define SMC_CLC_DECL_FEATUNSUPP 0x0303000c /* supplemental features not supported */
#define SMC_CLC_DECL_MODEUNSUPP 0x03040000 /* smc modes do not match (R or D)*/
#define SMC_CLC_DECL_RMBE_EC 0x03050000 /* peer has eyecatcher in RMBE */
#define SMC_CLC_DECL_OPTUNSUPP 0x03060000 /* fastopen sockopt not supported */
@@ -138,7 +139,8 @@ struct smc_clc_v2_extension {
u8 roce[16]; /* RoCEv2 GID */
u8 max_conns;
u8 max_links;
- u8 reserved[14];
+ __be16 feature_mask;
+ u8 reserved[12];
u8 user_eids[][SMC_MAX_EID_LEN];
};
@@ -234,7 +236,7 @@ struct smc_clc_first_contact_ext {
u8 release : 4,
os_type : 4;
#endif
- u8 reserved2[2];
+ __be16 feature_mask;
u8 hostname[SMC_MAX_HOSTNAME_LEN];
};
diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 120027d..9f65678 100644
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -401,6 +401,7 @@ struct smc_init_info {
u8 max_links;
u8 first_contact_peer;
u8 first_contact_local;
+ u16 feature_mask;
unsigned short vlan_id;
u32 rc;
u8 negotiated_eid[SMC_MAX_EID_LEN];
--
1.8.3.1
As the SMC-Dv2 protocol introduces virtual ISM devices, whose GIDs
are UUIDs generated by software, the maximum length of SMC-D GID has
been extended to 128 bits.
So this patch adapts the relevant code to make it compatible with
128 bits GID.
Signed-off-by: Wen Gu <[email protected]>
---
drivers/s390/net/ism_drv.c | 11 ++++++-----
include/net/smc.h | 11 ++++++++---
include/uapi/linux/smc.h | 3 +++
net/smc/af_smc.c | 45 +++++++++++++++++++++++++++++++++------------
net/smc/smc_clc.c | 35 ++++++++++++++++++++++++-----------
net/smc/smc_clc.h | 4 ++--
net/smc/smc_core.c | 38 +++++++++++++++++++++++++++++---------
net/smc/smc_core.h | 7 ++++---
net/smc/smc_diag.c | 6 ++++--
net/smc/smc_ism.c | 13 +++++++++----
net/smc/smc_ism.h | 3 ++-
net/smc/smc_pnet.c | 4 ++--
12 files changed, 126 insertions(+), 54 deletions(-)
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index a34e913..1b714f6 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -774,10 +774,10 @@ static void __exit ism_exit(void)
/*************************** SMC-D Implementation *****************************/
#if IS_ENABLED(CONFIG_SMC)
-static int smcd_query_rgid(struct smcd_dev *smcd, u64 rgid, u32 vid_valid,
- u32 vid)
+static int smcd_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
+ u32 vid_valid, u32 vid)
{
- return ism_query_rgid(smcd->priv, rgid, vid_valid, vid);
+ return ism_query_rgid(smcd->priv, rgid->gid, vid_valid, vid);
}
static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
@@ -830,9 +830,10 @@ static int smcd_supports_v2(void)
SYSTEM_EID.type[0] != '0';
}
-static u64 smcd_get_local_gid(struct smcd_dev *smcd)
+static void smcd_get_local_gid(struct smcd_dev *smcd,
+ struct smcd_gid *smcd_gid)
{
- return ism_get_local_gid(smcd->priv);
+ smcd_gid->gid = ism_get_local_gid(smcd->priv);
}
static u16 smcd_get_chid(struct smcd_dev *smcd)
diff --git a/include/net/smc.h b/include/net/smc.h
index f75116e..a37fb38 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -51,9 +51,14 @@ struct smcd_dmb {
struct smcd_dev;
+struct smcd_gid {
+ u64 gid;
+ u64 gid_ext;
+};
+
struct smcd_ops {
- int (*query_remote_gid)(struct smcd_dev *dev, u64 rgid, u32 vid_valid,
- u32 vid);
+ int (*query_remote_gid)(struct smcd_dev *dev, struct smcd_gid *rgid,
+ u32 vid_valid, u32 vid);
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
void *client);
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
@@ -68,7 +73,7 @@ struct smcd_ops {
unsigned int size);
int (*supports_v2)(void);
u8* (*get_system_eid)(void);
- u64 (*get_local_gid)(struct smcd_dev *dev);
+ void (*get_local_gid)(struct smcd_dev *dev, struct smcd_gid *gid);
u16 (*get_chid)(struct smcd_dev *dev);
struct device* (*get_dev)(struct smcd_dev *dev);
};
diff --git a/include/uapi/linux/smc.h b/include/uapi/linux/smc.h
index 837fcd4..0d2f020 100644
--- a/include/uapi/linux/smc.h
+++ b/include/uapi/linux/smc.h
@@ -99,6 +99,9 @@ enum {
SMC_NLA_LGR_V2_OS, /* u8 */
SMC_NLA_LGR_V2_NEG_EID, /* string */
SMC_NLA_LGR_V2_PEER_HOST, /* string */
+ SMC_NLA_LGR_V2_PAD, /* flag */
+ SMC_NLA_LGR_V2_GID_EXT, /* u64 */
+ SMC_NLA_LGR_V2_PEER_GID_EXT, /* u64 */
__SMC_NLA_LGR_V2_MAX,
SMC_NLA_LGR_V2_MAX = __SMC_NLA_LGR_V2_MAX - 1
};
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index bacdd97..9e31033 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -1044,7 +1044,8 @@ static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
{
int rc = SMC_CLC_DECL_NOSMCDDEV;
struct smcd_dev *smcd;
- int i = 1;
+ int i = 1, slot = 1;
+ bool is_virtdev;
u16 chid;
if (smcd_indicated(ini->smc_type_v1))
@@ -1056,14 +1057,19 @@ static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
chid = smc_ism_get_chid(smcd);
if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
continue;
+ is_virtdev = __smc_ism_is_virtdev(chid);
if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
+ /* GID-CHID array is not enough for a virtual GID (128bits) */
+ if (is_virtdev && slot >= SMC_MAX_ISM_DEVS)
+ continue;
ini->ism_dev[i] = smcd;
ini->ism_chid[i] = chid;
ini->is_smcd = true;
rc = 0;
i++;
- if (i > SMC_MAX_ISM_DEVS)
+ slot = is_virtdev ? slot + 2 : slot + 1;
+ if (slot > SMC_MAX_ISM_DEVS)
break;
}
}
@@ -1409,8 +1415,9 @@ static int smc_connect_ism(struct smc_sock *smc,
rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
if (rc)
return rc;
+ ini->ism_peer_gid[ini->ism_selected].gid_ext = aclc_v2->d1.gid_ext;
}
- ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
+ ini->ism_peer_gid[ini->ism_selected].gid = aclc->d0.gid;
/* there is only one lgr role for SMC-D; use server lock */
mutex_lock(&smc_server_lgr_pending);
@@ -2101,7 +2108,8 @@ static bool smc_is_already_selected(struct smcd_dev *smcd,
/* check for ISM devices matching proposed ISM devices */
static void smc_check_ism_v2_match(struct smc_init_info *ini,
- u16 proposed_chid, u64 proposed_gid,
+ u16 proposed_chid,
+ struct smcd_gid *proposed_gid,
unsigned int *matches)
{
struct smcd_dev *smcd;
@@ -2113,7 +2121,10 @@ static void smc_check_ism_v2_match(struct smc_init_info *ini,
continue;
if (smc_ism_get_chid(smcd) == proposed_chid &&
!smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
- ini->ism_peer_gid[*matches] = proposed_gid;
+ ini->ism_peer_gid[*matches].gid = proposed_gid->gid;
+ if (__smc_ism_is_virtdev(proposed_chid))
+ ini->ism_peer_gid[*matches].gid_ext =
+ proposed_gid->gid_ext;
ini->ism_dev[*matches] = smcd;
(*matches)++;
break;
@@ -2135,9 +2146,11 @@ static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
struct smc_clc_v2_extension *smc_v2_ext;
struct smc_clc_msg_smcd *pclc_smcd;
unsigned int matches = 0;
+ struct smcd_gid smcd_gid;
u8 smcd_version;
u8 *eid = NULL;
int i, rc;
+ u16 chid;
if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
goto not_found;
@@ -2147,18 +2160,26 @@ static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
mutex_lock(&smcd_dev_list.mutex);
- if (pclc_smcd->ism.chid)
+ if (pclc_smcd->ism.chid) {
/* check for ISM device matching proposed native ISM device */
+ smcd_gid.gid = ntohll(pclc_smcd->ism.gid);
smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
- ntohll(pclc_smcd->ism.gid), &matches);
+ &smcd_gid, &matches);
+ }
for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
/* check for ISM devices matching proposed non-native ISM
* devices
*/
- smc_check_ism_v2_match(ini,
- ntohs(smcd_v2_ext->gidchid[i - 1].chid),
- ntohll(smcd_v2_ext->gidchid[i - 1].gid),
- &matches);
+ smcd_gid.gid = ntohll(smcd_v2_ext->gidchid[i - 1].gid);
+ chid = ntohs(smcd_v2_ext->gidchid[i - 1].chid);
+ if (__smc_ism_is_virtdev(chid)) {
+ /* check if extended entry exists and is valid */
+ if (i >= smc_v2_ext->hdr.ism_gid_cnt ||
+ chid != ntohs(smcd_v2_ext->gidchid[i].chid))
+ continue;
+ smcd_gid.gid_ext = ntohll(smcd_v2_ext->gidchid[i++].gid);
+ }
+ smc_check_ism_v2_match(ini, chid, &smcd_gid, &matches);
}
mutex_unlock(&smcd_dev_list.mutex);
@@ -2207,7 +2228,7 @@ static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
goto not_found;
ini->is_smcd = true; /* prepare ISM check */
- ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
+ ini->ism_peer_gid[0].gid = ntohll(pclc_smcd->ism.gid);
rc = smc_find_ism_device(new_smc, ini);
if (rc)
goto not_found;
diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c
index 125b0d2..c08e138 100644
--- a/net/smc/smc_clc.c
+++ b/net/smc/smc_clc.c
@@ -882,11 +882,13 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
ETH_ALEN);
}
if (smcd_indicated(ini->smc_type_v1)) {
+ struct smcd_gid smcd_gid;
+
/* add SMC-D specifics */
if (ini->ism_dev[0]) {
smcd = ini->ism_dev[0];
- pclc_smcd->ism.gid =
- htonll(smcd->ops->get_local_gid(smcd));
+ smcd->ops->get_local_gid(smcd, &smcd_gid);
+ pclc_smcd->ism.gid = htonll(smcd_gid.gid);
pclc_smcd->ism.chid =
htons(smc_ism_get_chid(ini->ism_dev[0]));
}
@@ -919,10 +921,11 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
read_unlock(&smc_clc_eid_table.lock);
}
if (smcd_indicated(ini->smc_type_v2)) {
+ struct smcd_gid smcd_gid;
u8 *eid = NULL;
+ int slot = 0;
v2_ext->hdr.flag.seid = smc_clc_eid_table.seid_enabled;
- v2_ext->hdr.ism_gid_cnt = ini->ism_offered_cnt;
v2_ext->hdr.smcd_v2_ext_offset = htons(sizeof(*v2_ext) -
offsetofend(struct smc_clnt_opts_area_hdr,
smcd_v2_ext_offset) +
@@ -934,14 +937,21 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
if (ini->ism_offered_cnt) {
for (i = 1; i <= ini->ism_offered_cnt; i++) {
smcd = ini->ism_dev[i];
- gidchids[i - 1].gid =
- htonll(smcd->ops->get_local_gid(smcd));
- gidchids[i - 1].chid =
+ smcd->ops->get_local_gid(smcd, &smcd_gid);
+ gidchids[slot].chid =
htons(smc_ism_get_chid(ini->ism_dev[i]));
+ gidchids[slot].gid = htonll(smcd_gid.gid);
+ /* virtual-ism takes two slots */
+ if (__smc_ism_is_virtdev(gidchids[slot].chid)) {
+ gidchids[slot + 1].chid = gidchids[slot].chid;
+ gidchids[slot + 1].gid = htonll(smcd_gid.gid_ext);
+ slot++;
+ }
+ slot++;
}
- plen += ini->ism_offered_cnt *
- sizeof(struct smc_clc_smcd_gid_chid);
+ plen += slot * sizeof(struct smc_clc_smcd_gid_chid);
}
+ v2_ext->hdr.ism_gid_cnt = slot;
}
if (smcr_indicated(ini->smc_type_v2)) {
memcpy(v2_ext->roce, ini->smcrv2.ib_gid_v2, SMC_GID_SIZE);
@@ -977,7 +987,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
vec[i++].iov_len = sizeof(*smcd_v2_ext);
if (ini->ism_offered_cnt) {
vec[i].iov_base = gidchids;
- vec[i++].iov_len = ini->ism_offered_cnt *
+ vec[i++].iov_len = v2_ext->hdr.ism_gid_cnt *
sizeof(struct smc_clc_smcd_gid_chid);
}
}
@@ -1019,12 +1029,14 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc,
if (first_contact)
clc->hdr.typev2 |= SMC_FIRST_CONTACT_MASK;
if (conn->lgr->is_smcd) {
+ struct smcd_gid smcd_gid;
+
/* SMC-D specific settings */
memcpy(clc->hdr.eyecatcher, SMCD_EYECATCHER,
sizeof(SMCD_EYECATCHER));
+ conn->lgr->smcd->ops->get_local_gid(conn->lgr->smcd, &smcd_gid);
clc->hdr.typev1 = SMC_TYPE_D;
- clc->d0.gid =
- conn->lgr->smcd->ops->get_local_gid(conn->lgr->smcd);
+ clc->d0.gid = smcd_gid.gid;
clc->d0.token = conn->rmb_desc->token;
clc->d0.dmbe_size = conn->rmbe_size_comp;
clc->d0.dmbe_idx = 0;
@@ -1036,6 +1048,7 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc,
htons(smc_ism_get_chid(conn->lgr->smcd));
if (eid && eid[0])
memcpy(clc_v2->d1.eid, eid, SMC_MAX_EID_LEN);
+ clc_v2->d1.gid_ext = smcd_gid.gid_ext;
len = SMCD_CLC_ACCEPT_CONFIRM_LEN_V2;
if (first_contact) {
fce_len = smc_clc_fill_fce(&fce, ini);
diff --git a/net/smc/smc_clc.h b/net/smc/smc_clc.h
index bcf37c8..611763a 100644
--- a/net/smc/smc_clc.h
+++ b/net/smc/smc_clc.h
@@ -281,8 +281,8 @@ struct smc_clc_msg_accept_confirm_v2 { /* clc accept / confirm message */
struct smcd_clc_msg_accept_confirm_common d0;
__be16 chid;
u8 eid[SMC_MAX_EID_LEN];
- u8 reserved5[8];
- } d1;
+ __be64 gid_ext;
+ } __packed d1;
};
};
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index d520ee6..c36500a 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -284,6 +284,9 @@ static int smc_nl_fill_lgr_v2_common(struct smc_link_group *lgr,
{
char smc_host[SMC_MAX_HOSTNAME_LEN + 1];
char smc_eid[SMC_MAX_EID_LEN + 1];
+ struct smcd_dev *smcd = lgr->smcd;
+ struct smcd_gid smcd_gid;
+ bool is_virtdev;
if (nla_put_u8(skb, SMC_NLA_LGR_V2_VER, lgr->smc_version))
goto errv2attr;
@@ -299,6 +302,16 @@ static int smc_nl_fill_lgr_v2_common(struct smc_link_group *lgr,
smc_eid[SMC_MAX_EID_LEN] = 0;
if (nla_put_string(skb, SMC_NLA_LGR_V2_NEG_EID, smc_eid))
goto errv2attr;
+ smcd->ops->get_local_gid(smcd, &smcd_gid);
+ is_virtdev = smc_ism_is_virtdev(smcd);
+ if (nla_put_u64_64bit(skb, SMC_NLA_LGR_V2_GID_EXT,
+ is_virtdev ? smcd_gid.gid_ext : 0,
+ SMC_NLA_LGR_V2_PAD))
+ goto errv2attr;
+ if (nla_put_u64_64bit(skb, SMC_NLA_LGR_V2_PEER_GID_EXT,
+ is_virtdev ? lgr->peer_gid.gid_ext : 0,
+ SMC_NLA_LGR_V2_PAD))
+ goto errv2attr;
nla_nest_end(skb, v2_attrs);
return 0;
@@ -506,6 +519,7 @@ static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr,
{
char smc_pnet[SMC_MAX_PNETID_LEN + 1];
struct smcd_dev *smcd = lgr->smcd;
+ struct smcd_gid smcd_gid;
struct nlattr *attrs;
void *nlh;
@@ -521,11 +535,11 @@ static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr,
if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id)))
goto errattr;
+ smcd->ops->get_local_gid(smcd, &smcd_gid);
if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_GID,
- smcd->ops->get_local_gid(smcd),
- SMC_NLA_LGR_D_PAD))
+ smcd_gid.gid, SMC_NLA_LGR_D_PAD))
goto errattr;
- if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid,
+ if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid.gid,
SMC_NLA_LGR_D_PAD))
goto errattr;
if (nla_put_u8(skb, SMC_NLA_LGR_D_VLAN_ID, lgr->vlan_id))
@@ -1514,7 +1528,8 @@ void smc_lgr_terminate_sched(struct smc_link_group *lgr)
}
/* Called when peer lgr shutdown (regularly or abnormally) is received */
-void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
+void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid,
+ unsigned short vlan)
{
struct smc_link_group *lgr, *l;
LIST_HEAD(lgr_free_list);
@@ -1522,7 +1537,10 @@ void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
/* run common cleanup function and build free list */
spin_lock_bh(&dev->lgr_lock);
list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) {
- if ((!peer_gid || lgr->peer_gid == peer_gid) &&
+ if ((!peer_gid->gid ||
+ (lgr->peer_gid.gid == peer_gid->gid &&
+ !smc_ism_is_virtdev(dev) ? 1 :
+ lgr->peer_gid.gid_ext == peer_gid->gid_ext)) &&
(vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) {
if (peer_gid) /* peer triggered termination */
lgr->peer_shutdown = 1;
@@ -1859,10 +1877,12 @@ static bool smcr_lgr_match(struct smc_link_group *lgr, u8 smcr_version,
return false;
}
-static bool smcd_lgr_match(struct smc_link_group *lgr,
- struct smcd_dev *smcismdev, u64 peer_gid)
+static bool smcd_lgr_match(struct smc_link_group *lgr, struct smcd_dev *smcismdev,
+ struct smcd_gid *peer_gid)
{
- return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;
+ return lgr->peer_gid.gid == peer_gid->gid && lgr->smcd == smcismdev &&
+ smc_ism_is_virtdev(smcismdev) ?
+ (lgr->peer_gid.gid_ext == peer_gid->gid_ext) : 1;
}
/* create a new SMC connection (and a new link group if necessary) */
@@ -1892,7 +1912,7 @@ int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini)
write_lock_bh(&lgr->conns_lock);
if ((ini->is_smcd ?
smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected],
- ini->ism_peer_gid[ini->ism_selected]) :
+ &ini->ism_peer_gid[ini->ism_selected]) :
smcr_lgr_match(lgr, ini->smcr_version,
ini->peer_systemid,
ini->peer_gid, ini->peer_mac, role,
diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 9f65678..d57eb9b 100644
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -17,6 +17,7 @@
#include <linux/pci.h>
#include <rdma/ib_verbs.h>
#include <net/genetlink.h>
+#include <net/smc.h>
#include "smc.h"
#include "smc_ib.h"
@@ -355,7 +356,7 @@ struct smc_link_group {
/* max links can be added in lgr */
};
struct { /* SMC-D */
- u64 peer_gid;
+ struct smcd_gid peer_gid;
/* Peer GID (remote) */
struct smcd_dev *smcd;
/* ISM device for VLAN reg. */
@@ -417,7 +418,7 @@ struct smc_init_info {
u32 ib_clcqpn;
struct smc_init_info_smcrv2 smcrv2;
/* SMC-D */
- u64 ism_peer_gid[SMC_MAX_ISM_DEVS + 1];
+ struct smcd_gid ism_peer_gid[SMC_MAX_ISM_DEVS + 1];
struct smcd_dev *ism_dev[SMC_MAX_ISM_DEVS + 1];
u16 ism_chid[SMC_MAX_ISM_DEVS + 1];
u8 ism_offered_cnt; /* # of ISM devices offered */
@@ -545,7 +546,7 @@ static inline void smc_set_pci_values(struct pci_dev *pci_dev,
void smc_lgr_put(struct smc_link_group *lgr);
void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport);
void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport);
-void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid,
+void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid,
unsigned short vlan);
void smc_smcd_terminate_all(struct smcd_dev *dev);
void smc_smcr_terminate_all(struct smc_ib_device *smcibdev);
diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c
index 7ff2152..9c465cc 100644
--- a/net/smc/smc_diag.c
+++ b/net/smc/smc_diag.c
@@ -168,12 +168,14 @@ static int __smc_diag_dump(struct sock *sk, struct sk_buff *skb,
struct smc_connection *conn = &smc->conn;
struct smcd_diag_dmbinfo dinfo;
struct smcd_dev *smcd = conn->lgr->smcd;
+ struct smcd_gid smcd_gid;
memset(&dinfo, 0, sizeof(dinfo));
dinfo.linkid = *((u32 *)conn->lgr->id);
- dinfo.peer_gid = conn->lgr->peer_gid;
- dinfo.my_gid = smcd->ops->get_local_gid(smcd);
+ dinfo.peer_gid = conn->lgr->peer_gid.gid;
+ smcd->ops->get_local_gid(smcd, &smcd_gid);
+ dinfo.my_gid = smcd_gid.gid;
dinfo.token = conn->rmb_desc->token;
dinfo.peer_token = conn->peer_token;
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 6fa87d8..ba37b27 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -44,7 +44,8 @@ static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
#endif
/* Test if an ISM communication is possible - same CPC */
-int smc_ism_cantalk(u64 peer_gid, unsigned short vlan_id, struct smcd_dev *smcd)
+int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id,
+ struct smcd_dev *smcd)
{
return smcd->ops->query_remote_gid(smcd, peer_gid, vlan_id ? 1 : 0,
vlan_id);
@@ -223,7 +224,7 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
dmb.dmb_len = dmb_len;
dmb.sba_idx = dmb_desc->sba_idx;
dmb.vlan_id = lgr->vlan_id;
- dmb.rgid = lgr->peer_gid;
+ dmb.rgid = lgr->peer_gid.gid;
rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, lgr->smcd->client);
if (!rc) {
dmb_desc->sba_idx = dmb.sba_idx;
@@ -355,11 +356,13 @@ struct smc_ism_event_work {
static void smcd_handle_sw_event(struct smc_ism_event_work *wrk)
{
union smcd_sw_event_info ev_info;
+ struct smcd_gid peer_gid;
ev_info.info = wrk->event.info;
switch (wrk->event.code) {
case ISM_EVENT_CODE_SHUTDOWN: /* Peer shut down DMBs */
- smc_smcd_terminate(wrk->smcd, wrk->event.tok, ev_info.vlan_id);
+ peer_gid.gid = wrk->event.tok;
+ smc_smcd_terminate(wrk->smcd, &peer_gid, ev_info.vlan_id);
break;
case ISM_EVENT_CODE_TESTLINK: /* Activity timer */
if (ev_info.code == ISM_EVENT_REQUEST) {
@@ -379,10 +382,12 @@ static void smc_ism_event_work(struct work_struct *work)
{
struct smc_ism_event_work *wrk =
container_of(work, struct smc_ism_event_work, work);
+ struct smcd_gid smcd_gid;
switch (wrk->event.type) {
case ISM_EVENT_GID: /* GID event, token is peer GID */
- smc_smcd_terminate(wrk->smcd, wrk->event.tok, VLAN_VID_MASK);
+ smcd_gid.gid = wrk->event.tok;
+ smc_smcd_terminate(wrk->smcd, &smcd_gid, VLAN_VID_MASK);
break;
case ISM_EVENT_DMB:
break;
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index 2ecc8de..e6ea08c 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -33,7 +33,8 @@ struct smc_ism_vlanid { /* VLAN id set on ISM device */
struct smcd_dev;
-int smc_ism_cantalk(u64 peer_gid, unsigned short vlan_id, struct smcd_dev *dev);
+int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id,
+ struct smcd_dev *dev);
void smc_ism_set_conn(struct smc_connection *conn);
void smc_ism_unset_conn(struct smc_connection *conn);
int smc_ism_get_vlan(struct smcd_dev *dev, unsigned short vlan_id);
diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c
index 1177540..9f2c58c 100644
--- a/net/smc/smc_pnet.c
+++ b/net/smc/smc_pnet.c
@@ -1103,8 +1103,8 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
list_for_each_entry(ismdev, &smcd_dev_list.list, list) {
if (smc_pnet_match(ismdev->pnetid, ndev_pnetid) &&
!ismdev->going_away &&
- (!ini->ism_peer_gid[0] ||
- !smc_ism_cantalk(ini->ism_peer_gid[0], ini->vlan_id,
+ (!ini->ism_peer_gid[0].gid ||
+ !smc_ism_cantalk(&ini->ism_peer_gid[0], ini->vlan_id,
ismdev))) {
ini->ism_dev[0] = ismdev;
break;
--
1.8.3.1
This patch implements DMB registration, unregistration and data move
operations of SMC-D loopback.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_cdc.c | 6 +++
net/smc/smc_cdc.h | 1 +
net/smc/smc_loopback.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++--
net/smc/smc_loopback.h | 13 +++++
4 files changed, 145 insertions(+), 3 deletions(-)
diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c
index 89105e9..2641800 100644
--- a/net/smc/smc_cdc.c
+++ b/net/smc/smc_cdc.c
@@ -411,6 +411,12 @@ static void smc_cdc_msg_recv(struct smc_sock *smc, struct smc_cdc_msg *cdc)
static void smcd_cdc_rx_tsklet(struct tasklet_struct *t)
{
struct smc_connection *conn = from_tasklet(conn, t, rx_tsklet);
+
+ smcd_cdc_rx_handler(conn);
+}
+
+void smcd_cdc_rx_handler(struct smc_connection *conn)
+{
struct smcd_cdc_msg *data_cdc;
struct smcd_cdc_msg cdc;
struct smc_sock *smc;
diff --git a/net/smc/smc_cdc.h b/net/smc/smc_cdc.h
index 696cc11..11559d4 100644
--- a/net/smc/smc_cdc.h
+++ b/net/smc/smc_cdc.h
@@ -301,5 +301,6 @@ int smcr_cdc_msg_send_validation(struct smc_connection *conn,
struct smc_wr_buf *wr_buf);
int smc_cdc_init(void) __init;
void smcd_cdc_rx_init(struct smc_connection *conn);
+void smcd_cdc_rx_handler(struct smc_connection *conn);
#endif /* SMC_CDC_H */
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index fe61260..7807b38 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -15,6 +15,7 @@
#include <linux/types.h>
#include <net/smc.h>
+#include "smc_cdc.h"
#include "smc_ism.h"
#include "smc_loopback.h"
@@ -73,6 +74,93 @@ static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
return 0;
}
+static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
+ void *client_priv)
+{
+ struct smc_lo_dmb_node *dmb_node, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+ int sba_idx, rc;
+
+ /* check space for new dmb */
+ for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LODEV_MAX_DMBS) {
+ if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
+ break;
+ }
+ if (sba_idx == SMC_LODEV_MAX_DMBS)
+ return -ENOSPC;
+
+ dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
+ if (!dmb_node) {
+ rc = -ENOMEM;
+ goto err_bit;
+ }
+
+ dmb_node->sba_idx = sba_idx;
+ dmb_node->cpu_addr = kzalloc(dmb->dmb_len, GFP_KERNEL |
+ __GFP_NOWARN | __GFP_NORETRY |
+ __GFP_NOMEMALLOC);
+ if (!dmb_node->cpu_addr) {
+ rc = -ENOMEM;
+ goto err_node;
+ }
+ dmb_node->len = dmb->dmb_len;
+ dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
+
+again:
+ /* add new dmb into hash table */
+ get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
+ write_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
+ if (tmp_node->token == dmb_node->token) {
+ write_unlock(&ldev->dmb_ht_lock);
+ goto again;
+ }
+ }
+ hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
+ write_unlock(&ldev->dmb_ht_lock);
+
+ dmb->sba_idx = dmb_node->sba_idx;
+ dmb->dmb_tok = dmb_node->token;
+ dmb->cpu_addr = dmb_node->cpu_addr;
+ dmb->dma_addr = dmb_node->dma_addr;
+ dmb->dmb_len = dmb_node->len;
+
+ return 0;
+
+err_node:
+ kfree(dmb_node);
+err_bit:
+ clear_bit(sba_idx, ldev->sba_idx_mask);
+ return rc;
+}
+
+static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
+{
+ struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* remove dmb from hash table */
+ write_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
+ if (tmp_node->token == dmb->dmb_tok) {
+ dmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!dmb_node) {
+ write_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ hash_del(&dmb_node->list);
+ write_unlock(&ldev->dmb_ht_lock);
+
+ clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
+ kfree(dmb_node->cpu_addr);
+ kfree(dmb_node);
+
+ return 0;
+}
+
static int smc_lo_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
{
return -EOPNOTSUPP;
@@ -99,6 +187,38 @@ static int smc_lo_signal_event(struct smcd_dev *dev, u64 rgid, u32 trigger_irq,
return 0;
}
+static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
+ bool sf, unsigned int offset, void *data,
+ unsigned int size)
+{
+ struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ read_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
+ if (tmp_node->token == dmb_tok) {
+ rmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!rmb_node) {
+ read_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ read_unlock(&ldev->dmb_ht_lock);
+
+ memcpy((char *)rmb_node->cpu_addr + offset, data, size);
+
+ if (sf) {
+ struct smc_connection *conn =
+ smcd->conn[rmb_node->sba_idx];
+
+ if (conn && !conn->killed)
+ smcd_cdc_rx_handler(conn);
+ }
+ return 0;
+}
+
static int smc_lo_supports_v2(void)
{
return SMC_LO_SUPPORTS_V2;
@@ -130,14 +250,14 @@ static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
static const struct smcd_ops lo_ops = {
.query_remote_gid = smc_lo_query_rgid,
- .register_dmb = NULL,
- .unregister_dmb = NULL,
+ .register_dmb = smc_lo_register_dmb,
+ .unregister_dmb = smc_lo_unregister_dmb,
.add_vlan_id = smc_lo_add_vlan_id,
.del_vlan_id = smc_lo_del_vlan_id,
.set_vlan_required = smc_lo_set_vlan_required,
.reset_vlan_required = smc_lo_reset_vlan_required,
.signal_event = smc_lo_signal_event,
- .move_data = NULL,
+ .move_data = smc_lo_move_data,
.supports_v2 = smc_lo_supports_v2,
.get_system_eid = smc_lo_get_system_eid,
.get_local_gid = smc_lo_get_local_gid,
@@ -211,6 +331,8 @@ static void smc_lo_dev_release(struct device *dev)
static int smc_lo_dev_init(struct smc_lo_dev *ldev)
{
smc_lo_generate_id(ldev);
+ rwlock_init(&ldev->dmb_ht_lock);
+ hash_init(ldev->dmb_ht);
return smcd_lo_register_dev(ldev);
}
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index 2156f22..943424f 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -20,12 +20,25 @@
#define SMC_LO_CHID 0xFFFF
#define SMC_LODEV_MAX_DMBS 5000
+#define SMC_LODEV_DMBS_HASH_BITS 12
+
+struct smc_lo_dmb_node {
+ struct hlist_node list;
+ u64 token;
+ u32 len;
+ u32 sba_idx;
+ void *cpu_addr;
+ dma_addr_t dma_addr;
+};
struct smc_lo_dev {
struct smcd_dev *smcd;
struct device dev;
u16 chid;
struct smcd_gid local_gid;
+ DECLARE_BITMAP(sba_idx_mask, SMC_LODEV_MAX_DMBS);
+ rwlock_t dmb_ht_lock;
+ DECLARE_HASHTABLE(dmb_ht, SMC_LODEV_DMBS_HASH_BITS);
};
int smc_loopback_init(void);
--
1.8.3.1
This patch completes the specific implementation of loopback device
for the newly added SMC-D DMB-related interface.
The loopback device always provides mappable DMB because the device
users are in the same OS instance.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_loopback.c | 105 ++++++++++++++++++++++++++++++++++++++++++++-----
net/smc/smc_loopback.h | 5 +++
2 files changed, 100 insertions(+), 10 deletions(-)
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 650561b..611998b 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -105,6 +105,7 @@ static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
}
dmb_node->len = dmb->dmb_len;
dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
+ refcount_set(&dmb_node->refcnt, 1);
again:
/* add new dmb into hash table */
@@ -118,6 +119,7 @@ static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
}
hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
write_unlock(&ldev->dmb_ht_lock);
+ atomic_inc(&ldev->dmb_cnt);
dmb->sba_idx = dmb_node->sba_idx;
dmb->dmb_tok = dmb_node->token;
@@ -139,28 +141,98 @@ static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
struct smc_lo_dev *ldev = smcd->priv;
- /* remove dmb from hash table */
- write_lock(&ldev->dmb_ht_lock);
+ /* find dmb from hash table */
+ read_lock(&ldev->dmb_ht_lock);
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
if (tmp_node->token == dmb->dmb_tok) {
dmb_node = tmp_node;
+ dmb_node->freeing = 1;
break;
}
}
if (!dmb_node) {
- write_unlock(&ldev->dmb_ht_lock);
+ read_unlock(&ldev->dmb_ht_lock);
return -EINVAL;
}
+ read_unlock(&ldev->dmb_ht_lock);
+
+ /* wait for dmb refcnt to be 0 */
+ if (!refcount_dec_and_test(&dmb_node->refcnt))
+ wait_event(ldev->dmbs_release, !refcount_read(&dmb_node->refcnt));
+
+ /* remove dmb from hash table */
+ write_lock(&ldev->dmb_ht_lock);
hash_del(&dmb_node->list);
write_unlock(&ldev->dmb_ht_lock);
clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
+
kfree(dmb_node->cpu_addr);
kfree(dmb_node);
+ if (atomic_dec_and_test(&ldev->dmb_cnt))
+ wake_up(&ldev->ldev_release);
+ return 0;
+}
+
+static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
+{
+ struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* find dmb_node according to dmb->dmb_tok */
+ read_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
+ if (tmp_node->token == dmb->dmb_tok && !tmp_node->freeing) {
+ dmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!dmb_node) {
+ read_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ refcount_inc(&dmb_node->refcnt);
+ read_unlock(&ldev->dmb_ht_lock);
+
+ /* provide dmb information */
+ dmb->sba_idx = dmb_node->sba_idx;
+ dmb->dmb_tok = dmb_node->token;
+ dmb->cpu_addr = dmb_node->cpu_addr;
+ dmb->dma_addr = dmb_node->dma_addr;
+ dmb->dmb_len = dmb_node->len;
return 0;
}
+static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token)
+{
+ struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
+ struct smc_lo_dev *ldev = smcd->priv;
+
+ /* find dmb_node according to dmb->dmb_tok */
+ read_lock(&ldev->dmb_ht_lock);
+ hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
+ if (tmp_node->token == token) {
+ dmb_node = tmp_node;
+ break;
+ }
+ }
+ if (!dmb_node) {
+ read_unlock(&ldev->dmb_ht_lock);
+ return -EINVAL;
+ }
+ read_unlock(&ldev->dmb_ht_lock);
+
+ if (refcount_dec_and_test(&dmb_node->refcnt))
+ wake_up_all(&ldev->dmbs_release);
+ return 0;
+}
+
+static int smc_lo_get_dev_attr(struct smcd_dev *smcd)
+{
+ return BIT(ISM_ATTR_DMB_MAP);
+}
+
static int smc_lo_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
{
return -EOPNOTSUPP;
@@ -193,7 +265,15 @@ static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx
{
struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node;
struct smc_lo_dev *ldev = smcd->priv;
-
+ struct smc_connection *conn;
+
+ if (!sf) {
+ /* local sndbuf shares the same physical memory with
+ * peer RMB, so no need to copy data from local sndbuf
+ * to peer RMB.
+ */
+ return 0;
+ }
read_lock(&ldev->dmb_ht_lock);
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
if (tmp_node->token == dmb_tok) {
@@ -209,13 +289,10 @@ static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx
memcpy((char *)rmb_node->cpu_addr + offset, data, size);
- if (sf) {
- struct smc_connection *conn =
- smcd->conn[rmb_node->sba_idx];
+ conn = smcd->conn[rmb_node->sba_idx];
+ if (conn && !conn->killed)
+ smcd_cdc_rx_handler(conn);
- if (conn && !conn->killed)
- smcd_cdc_rx_handler(conn);
- }
return 0;
}
@@ -252,6 +329,8 @@ static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
.query_remote_gid = smc_lo_query_rgid,
.register_dmb = smc_lo_register_dmb,
.unregister_dmb = smc_lo_unregister_dmb,
+ .attach_dmb = smc_lo_attach_dmb,
+ .detach_dmb = smc_lo_detach_dmb,
.add_vlan_id = smc_lo_add_vlan_id,
.del_vlan_id = smc_lo_del_vlan_id,
.set_vlan_required = smc_lo_set_vlan_required,
@@ -263,6 +342,7 @@ static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
.get_local_gid = smc_lo_get_local_gid,
.get_chid = smc_lo_get_chid,
.get_dev = smc_lo_get_dev,
+ .get_dev_attr = smc_lo_get_dev_attr,
};
static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops,
@@ -342,6 +422,9 @@ static int smc_lo_dev_init(struct smc_lo_dev *ldev)
smc_lo_generate_id(ldev);
rwlock_init(&ldev->dmb_ht_lock);
hash_init(ldev->dmb_ht);
+ atomic_set(&ldev->dmb_cnt, 0);
+ init_waitqueue_head(&ldev->dmbs_release);
+ init_waitqueue_head(&ldev->ldev_release);
return smcd_lo_register_dev(ldev);
}
@@ -375,6 +458,8 @@ static int smc_lo_dev_probe(void)
static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
{
smcd_lo_unregister_dev(ldev);
+ if (atomic_read(&ldev->dmb_cnt))
+ wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
}
static void smc_lo_dev_remove(void)
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
index 943424f..506e524 100644
--- a/net/smc/smc_loopback.h
+++ b/net/smc/smc_loopback.h
@@ -29,6 +29,8 @@ struct smc_lo_dmb_node {
u32 sba_idx;
void *cpu_addr;
dma_addr_t dma_addr;
+ refcount_t refcnt;
+ u8 freeing : 1;
};
struct smc_lo_dev {
@@ -39,6 +41,9 @@ struct smc_lo_dev {
DECLARE_BITMAP(sba_idx_mask, SMC_LODEV_MAX_DMBS);
rwlock_t dmb_ht_lock;
DECLARE_HASHTABLE(dmb_ht, SMC_LODEV_DMBS_HASH_BITS);
+ atomic_t dmb_cnt;
+ wait_queue_head_t dmbs_release;
+ wait_queue_head_t ldev_release;
};
int smc_loopback_init(void);
--
1.8.3.1
This patch reserve CHID range from 0xFF00 to 0xFFFF for SMC-D virtual
device and introduces helpers to identify them.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_ism.h | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index 14d2e77..2ecc8de 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -15,6 +15,9 @@
#include "smc.h"
+#define SMC_VIRT_ISM_CHID_MAX 0xFFFF
+#define SMC_VIRT_ISM_CHID_MIN 0xFF00
+
struct smcd_dev_list { /* List of SMCD devices */
struct list_head list;
struct mutex mutex; /* Protects list of devices */
@@ -57,4 +60,16 @@ static inline int smc_ism_write(struct smcd_dev *smcd, u64 dmb_tok,
return rc < 0 ? rc : 0;
}
+static inline bool __smc_ism_is_virtdev(u16 chid)
+{
+ return (chid >= SMC_VIRT_ISM_CHID_MIN && chid <= SMC_VIRT_ISM_CHID_MAX);
+}
+
+static inline bool smc_ism_is_virtdev(struct smcd_dev *smcd)
+{
+ u16 chid = smcd->ops->get_chid(smcd);
+
+ return __smc_ism_is_virtdev(chid);
+}
+
#endif
--
1.8.3.1
On s390 architecture SMC-Dv2 SEID is generated by ISMv2 device with
partial cpuid. It is predefined and unique to represents the physical
machine (CPC), the maximum communication space of SMC-D.
On non-s390 archs like x86/64 or ARM, there is no similar information
to identify physical machine, especially in virtualization scenarios
like KVM, etc. In such cases, SEID is forcibly disabled and the user-
defined UEID will be used to represent the communicable space.
Signed-off-by: Wen Gu <[email protected]>
---
net/smc/smc_clc.c | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c
index c08e138..be56a7d 100644
--- a/net/smc/smc_clc.c
+++ b/net/smc/smc_clc.c
@@ -155,10 +155,12 @@ static int smc_clc_ueid_remove(char *ueid)
rc = 0;
}
}
+#if IS_ENABLED(CONFIG_S390)
if (!rc && !smc_clc_eid_table.ueid_cnt) {
smc_clc_eid_table.seid_enabled = 1;
rc = -EAGAIN; /* indicate success and enabling of seid */
}
+#endif
write_unlock(&smc_clc_eid_table.lock);
return rc;
}
@@ -273,22 +275,30 @@ int smc_nl_dump_seid(struct sk_buff *skb, struct netlink_callback *cb)
int smc_nl_enable_seid(struct sk_buff *skb, struct genl_info *info)
{
+#if IS_ENABLED(CONFIG_S390)
write_lock(&smc_clc_eid_table.lock);
smc_clc_eid_table.seid_enabled = 1;
write_unlock(&smc_clc_eid_table.lock);
return 0;
+#else
+ return -EOPNOTSUPP;
+#endif
}
int smc_nl_disable_seid(struct sk_buff *skb, struct genl_info *info)
{
int rc = 0;
+#if IS_ENABLED(CONFIG_S390)
write_lock(&smc_clc_eid_table.lock);
if (!smc_clc_eid_table.ueid_cnt)
rc = -ENOENT;
else
smc_clc_eid_table.seid_enabled = 0;
write_unlock(&smc_clc_eid_table.lock);
+#else
+ rc = -EOPNOTSUPP;
+#endif
return rc;
}
@@ -1290,7 +1300,11 @@ void __init smc_clc_init(void)
INIT_LIST_HEAD(&smc_clc_eid_table.list);
rwlock_init(&smc_clc_eid_table.lock);
smc_clc_eid_table.ueid_cnt = 0;
+#if IS_ENABLED(CONFIG_S390)
smc_clc_eid_table.seid_enabled = 1;
+#else
+ smc_clc_eid_table.seid_enabled = 0;
+#endif
}
void smc_clc_exit(void)
--
1.8.3.1
This patch tries to decouple ISM device from SMC-D DMB registration,
So that the register_dmb option is not restricted to ISM devices.
Signed-off-by: Wen Gu <[email protected]>
---
drivers/s390/net/ism_drv.c | 2 +-
include/net/smc.h | 4 ++--
net/smc/smc_ism.c | 7 ++-----
3 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index 6df7f37..a34e913 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -781,7 +781,7 @@ static int smcd_query_rgid(struct smcd_dev *smcd, u64 rgid, u32 vid_valid,
}
static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
- struct ism_client *client)
+ void *client)
{
return ism_register_dmb(smcd->priv, (struct ism_dmb *)dmb, client);
}
diff --git a/include/net/smc.h b/include/net/smc.h
index a002552..f75116e 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -50,13 +50,12 @@ struct smcd_dmb {
#define ISM_ERROR 0xFFFF
struct smcd_dev;
-struct ism_client;
struct smcd_ops {
int (*query_remote_gid)(struct smcd_dev *dev, u64 rgid, u32 vid_valid,
u32 vid);
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
- struct ism_client *client);
+ void *client);
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
int (*add_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*del_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
@@ -77,6 +76,7 @@ struct smcd_ops {
struct smcd_dev {
const struct smcd_ops *ops;
void *priv;
+ void *client;
struct list_head list;
spinlock_t lock;
struct smc_connection **conn;
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 0045fee..9e53bcf 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -200,7 +200,6 @@ int smc_ism_unregister_dmb(struct smcd_dev *smcd, struct smc_buf_desc *dmb_desc)
int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
struct smc_buf_desc *dmb_desc)
{
-#if IS_ENABLED(CONFIG_ISM)
struct smcd_dmb dmb;
int rc;
@@ -209,7 +208,7 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
dmb.sba_idx = dmb_desc->sba_idx;
dmb.vlan_id = lgr->vlan_id;
dmb.rgid = lgr->peer_gid;
- rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, &smc_ism_client);
+ rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, lgr->smcd->client);
if (!rc) {
dmb_desc->sba_idx = dmb.sba_idx;
dmb_desc->token = dmb.dmb_tok;
@@ -218,9 +217,6 @@ int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
dmb_desc->len = dmb.dmb_len;
}
return rc;
-#else
- return 0;
-#endif
}
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
@@ -422,6 +418,7 @@ static void smcd_register_dev(struct ism_dev *ism)
if (!smcd)
return;
smcd->priv = ism;
+ smcd->client = &smc_ism_client;
ism_set_priv(ism, &smc_ism_client, smcd);
if (smc_pnetid_by_dev_port(&ism->pdev->dev, 0, smcd->pnetid))
smc_pnetid_by_table_smcd(smcd);
--
1.8.3.1
On 2023/9/20 15:02, kernel test robot wrote:
> Hi Wen,
>
> kernel test robot noticed the following build errors:
>
> [auto build test ERROR on net-next/main]
>
> url: https://github.com/intel-lab-lkp/linux/commits/Wen-Gu/net-smc-decouple-ism_dev-from-SMC-D-device-dump/20230920-010019
> base: net-next/main
> patch link: https://lore.kernel.org/r/1695134522-126655-7-git-send-email-guwen%40linux.alibaba.com
> patch subject: [PATCH net-next 06/18] net/smc: extend GID to 128bits for virtual ISM device
> config: s390-defconfig (https://download.01.org/0day-ci/archive/20230920/[email protected]/config)
> compiler: s390-linux-gcc (GCC) 13.2.0
> reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230920/[email protected]/reproduce)
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <[email protected]>
> | Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
>
> All errors (new ones prefixed by >>):
>
> net/smc/smc_ism.c: In function 'smc_ism_signal_shutdown':
>>> net/smc/smc_ism.c:539:57: error: incompatible type for argument 2 of 'lgr-><U72f8>.<U7260>.smcd->ops->signal_event'
> 539 | rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
> | ~~~^~~~~~~~~~
> | |
> | struct smcd_gid
> net/smc/smc_ism.c:539:57: note: expected 'u64' {aka 'long long unsigned int'} but argument is of type 'struct smcd_gid'
>
>
> vim +539 net/smc/smc_ism.c
>
> 820f21009f1bc7a Stefan Raspl 2023-01-23 526
> 820f21009f1bc7a Stefan Raspl 2023-01-23 527 int smc_ism_signal_shutdown(struct smc_link_group *lgr)
> 820f21009f1bc7a Stefan Raspl 2023-01-23 528 {
> 820f21009f1bc7a Stefan Raspl 2023-01-23 529 int rc = 0;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 530 #if IS_ENABLED(CONFIG_ISM)
> 820f21009f1bc7a Stefan Raspl 2023-01-23 531 union smcd_sw_event_info ev_info;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 532
> 820f21009f1bc7a Stefan Raspl 2023-01-23 533 if (lgr->peer_shutdown)
> 820f21009f1bc7a Stefan Raspl 2023-01-23 534 return 0;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 535
> 820f21009f1bc7a Stefan Raspl 2023-01-23 536 memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE);
> 820f21009f1bc7a Stefan Raspl 2023-01-23 537 ev_info.vlan_id = lgr->vlan_id;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 538 ev_info.code = ISM_EVENT_REQUEST;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 @539 rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
> 820f21009f1bc7a Stefan Raspl 2023-01-23 540 ISM_EVENT_REQUEST_IR,
> 820f21009f1bc7a Stefan Raspl 2023-01-23 541 ISM_EVENT_CODE_SHUTDOWN,
> 820f21009f1bc7a Stefan Raspl 2023-01-23 542 ev_info.info);
> 820f21009f1bc7a Stefan Raspl 2023-01-23 543 #endif
> 820f21009f1bc7a Stefan Raspl 2023-01-23 544 return rc;
> 820f21009f1bc7a Stefan Raspl 2023-01-23 545 }
> 201091ebb2a161a Ursula Braun 2020-09-26 546
>
I do not have a local compilation environment for s390 (IBM Z) architecture. But I think
it can be fixed by the following patch.
After this v1 exceeds 24 hours, I will send a v2 with this fix. Thanks.
Fix:
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index 1b714f6..7801a6a 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -811,10 +811,10 @@ static int smcd_reset_vlan_required(struct smcd_dev *smcd)
return ism_cmd_simple(smcd->priv, ISM_RESET_VLAN);
}
-static int smcd_signal_ieq(struct smcd_dev *smcd, u64 rgid, u32 trigger_irq,
- u32 event_code, u64 info)
+static int smcd_signal_ieq(struct smcd_dev *smcd, struct smcd_gid *rgid,
+ u32 trigger_irq, u32 event_code, u64 info)
{
- return ism_signal_ieq(smcd->priv, rgid, trigger_irq, event_code, info);
+ return ism_signal_ieq(smcd->priv, rgid->gid, trigger_irq, event_code, info);
}
static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
diff --git a/include/net/smc.h b/include/net/smc.h
index 04dbaa4..39a21ed 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -78,8 +78,8 @@ struct smcd_ops {
int (*del_vlan_id)(struct smcd_dev *dev, u64 vlan_id);
int (*set_vlan_required)(struct smcd_dev *dev);
int (*reset_vlan_required)(struct smcd_dev *dev);
- int (*signal_event)(struct smcd_dev *dev, u64 rgid, u32 trigger_irq,
- u32 event_code, u64 info);
+ int (*signal_event)(struct smcd_dev *dev, struct smcd_gid *rgid,
+ u32 trigger_irq, u32 event_code, u64 info);
int (*move_data)(struct smcd_dev *dev, u64 dmb_tok, unsigned int idx,
bool sf, unsigned int offset, void *data,
unsigned int size);
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 0c20745..3d9588e 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -404,10 +404,11 @@ static void smcd_handle_sw_event(struct smc_ism_event_work *wrk)
smc_smcd_terminate(wrk->smcd, &peer_gid, ev_info.vlan_id);
break;
case ISM_EVENT_CODE_TESTLINK: /* Activity timer */
+ peer_gid.gid = wrk->event.tok;
if (ev_info.code == ISM_EVENT_REQUEST) {
ev_info.code = ISM_EVENT_RESPONSE;
wrk->smcd->ops->signal_event(wrk->smcd,
- wrk->event.tok,
+ &peer_gid,
ISM_EVENT_REQUEST_IR,
ISM_EVENT_CODE_TESTLINK,
ev_info.info);
@@ -575,7 +576,7 @@ int smc_ism_signal_shutdown(struct smc_link_group *lgr)
memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE);
ev_info.vlan_id = lgr->vlan_id;
ev_info.code = ISM_EVENT_REQUEST;
- rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
+ rc = lgr->smcd->ops->signal_event(lgr->smcd, &lgr->peer_gid,
ISM_EVENT_REQUEST_IR,
ISM_EVENT_CODE_SHUTDOWN,
ev_info.info);
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
index 611998b..8b249a5 100644
--- a/net/smc/smc_loopback.c
+++ b/net/smc/smc_loopback.c
@@ -253,8 +253,8 @@ static int smc_lo_reset_vlan_required(struct smcd_dev *smcd)
return -EOPNOTSUPP;
}
-static int smc_lo_signal_event(struct smcd_dev *dev, u64 rgid, u32 trigger_irq,
- u32 event_code, u64 info)
+static int smc_lo_signal_event(struct smcd_dev *dev, struct smcd_gid *rgid,
+ u32 trigger_irq, u32 event_code, u64 info)
{
return 0;
}
On Wed, 2023-09-20 at 16:11 +0800, Wen Gu wrote:
>
> On 2023/9/20 15:02, kernel test robot wrote:
> > Hi Wen,
> >
> > kernel test robot noticed the following build errors:
> >
> > [auto build test ERROR on net-next/main]
> >
> > url: https://github.com/intel-lab-lkp/linux/commits/Wen-Gu/net-smc-decouple-ism_dev-from-SMC-D-device-dump/20230920-010019
> > base: net-next/main
> > patch link: https://lore.kernel.org/r/1695134522-126655-7-git-send-email-guwen%40linux.alibaba.com
> > patch subject: [PATCH net-next 06/18] net/smc: extend GID to 128bits for virtual ISM device
> > config: s390-defconfig (https://download.01.org/0day-ci/archive/20230920/[email protected]/config)
> > compiler: s390-linux-gcc (GCC) 13.2.0
> > reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230920/[email protected]/reproduce)
> >
> > If you fix the issue in a separate patch/commit (i.e. not just a new version of
> > the same patch/commit), kindly add following tags
> > > Reported-by: kernel test robot <[email protected]>
> > > Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
> >
> > All errors (new ones prefixed by >>):
> >
> > net/smc/smc_ism.c: In function 'smc_ism_signal_shutdown':
> > > > net/smc/smc_ism.c:539:57: error: incompatible type for argument 2 of 'lgr-><U72f8>.<U7260>.smcd->ops->signal_event'
> > 539 | rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
> > | ~~~^~~~~~~~~~
> > | |
> > | struct smcd_gid
> > net/smc/smc_ism.c:539:57: note: expected 'u64' {aka 'long long unsigned int'} but argument is of type 'struct smcd_gid'
> >
> >
> > vim +539 net/smc/smc_ism.c
> >
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 526
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 527 int smc_ism_signal_shutdown(struct smc_link_group *lgr)
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 528 {
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 529 int rc = 0;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 530 #if IS_ENABLED(CONFIG_ISM)
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 531 union smcd_sw_event_info ev_info;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 532
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 533 if (lgr->peer_shutdown)
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 534 return 0;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 535
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 536 memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE);
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 537 ev_info.vlan_id = lgr->vlan_id;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 538 ev_info.code = ISM_EVENT_REQUEST;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 @539 rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 540 ISM_EVENT_REQUEST_IR,
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 541 ISM_EVENT_CODE_SHUTDOWN,
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 542 ev_info.info);
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 543 #endif
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 544 return rc;
> > 820f21009f1bc7a Stefan Raspl 2023-01-23 545 }
> > 201091ebb2a161a Ursula Braun 2020-09-26 546
> >
>
> I do not have a local compilation environment for s390 (IBM Z) architecture. But I think
> it can be fixed by the following patch.
With these kernel test robot mails the bot provides
instructions for reproducing with a cross toolchain from the 0day
project. See the line starting with "reproduce (this is a W=1 build):"
i.e. in this particular case it links the URL:
https://download.01.org/0day-ci/archive/20230920/[email protected]/reproduce
On 2023/9/20 17:00, Niklas Schnelle wrote:
>>
>> I do not have a local compilation environment for s390 (IBM Z) architecture. But I think
>> it can be fixed by the following patch.
>
> With these kernel test robot mails the bot provides
> instructions for reproducing with a cross toolchain from the 0day
> project. See the line starting with "reproduce (this is a W=1 build):"
> i.e. in this particular case it links the URL:
> https://download.01.org/0day-ci/archive/20230920/[email protected]/reproduce
Ah, I see. Thank you Niklas for the explanation. :)
Hi Wen,
kernel test robot noticed the following build errors:
[auto build test ERROR on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Wen-Gu/net-smc-decouple-ism_dev-from-SMC-D-device-dump/20230920-010019
base: net-next/main
patch link: https://lore.kernel.org/r/1695134522-126655-7-git-send-email-guwen%40linux.alibaba.com
patch subject: [PATCH net-next 06/18] net/smc: extend GID to 128bits for virtual ISM device
config: s390-defconfig (https://download.01.org/0day-ci/archive/20230920/[email protected]/config)
compiler: s390-linux-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230920/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
net/smc/smc_ism.c: In function 'smc_ism_signal_shutdown':
>> net/smc/smc_ism.c:539:57: error: incompatible type for argument 2 of 'lgr-><U72f8>.<U7260>.smcd->ops->signal_event'
539 | rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
| ~~~^~~~~~~~~~
| |
| struct smcd_gid
net/smc/smc_ism.c:539:57: note: expected 'u64' {aka 'long long unsigned int'} but argument is of type 'struct smcd_gid'
vim +539 net/smc/smc_ism.c
820f21009f1bc7a Stefan Raspl 2023-01-23 526
820f21009f1bc7a Stefan Raspl 2023-01-23 527 int smc_ism_signal_shutdown(struct smc_link_group *lgr)
820f21009f1bc7a Stefan Raspl 2023-01-23 528 {
820f21009f1bc7a Stefan Raspl 2023-01-23 529 int rc = 0;
820f21009f1bc7a Stefan Raspl 2023-01-23 530 #if IS_ENABLED(CONFIG_ISM)
820f21009f1bc7a Stefan Raspl 2023-01-23 531 union smcd_sw_event_info ev_info;
820f21009f1bc7a Stefan Raspl 2023-01-23 532
820f21009f1bc7a Stefan Raspl 2023-01-23 533 if (lgr->peer_shutdown)
820f21009f1bc7a Stefan Raspl 2023-01-23 534 return 0;
820f21009f1bc7a Stefan Raspl 2023-01-23 535
820f21009f1bc7a Stefan Raspl 2023-01-23 536 memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE);
820f21009f1bc7a Stefan Raspl 2023-01-23 537 ev_info.vlan_id = lgr->vlan_id;
820f21009f1bc7a Stefan Raspl 2023-01-23 538 ev_info.code = ISM_EVENT_REQUEST;
820f21009f1bc7a Stefan Raspl 2023-01-23 @539 rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid,
820f21009f1bc7a Stefan Raspl 2023-01-23 540 ISM_EVENT_REQUEST_IR,
820f21009f1bc7a Stefan Raspl 2023-01-23 541 ISM_EVENT_CODE_SHUTDOWN,
820f21009f1bc7a Stefan Raspl 2023-01-23 542 ev_info.info);
820f21009f1bc7a Stefan Raspl 2023-01-23 543 #endif
820f21009f1bc7a Stefan Raspl 2023-01-23 544 return rc;
820f21009f1bc7a Stefan Raspl 2023-01-23 545 }
201091ebb2a161a Ursula Braun 2020-09-26 546
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Wen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Wen-Gu/net-smc-decouple-ism_dev-from-SMC-D-device-dump/20230920-010019
base: net-next/main
patch link: https://lore.kernel.org/r/1695134522-126655-13-git-send-email-guwen%40linux.alibaba.com
patch subject: [PATCH net-next 12/18] net/smc: implement DMB-related operations of loopback
config: arc-allmodconfig (https://download.01.org/0day-ci/archive/20230923/[email protected]/config)
compiler: arceb-elf-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230923/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
net/smc/smc_loopback.c: In function 'smc_lo_register_dmb':
>> net/smc/smc_loopback.c:107:30: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
107 | dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
| ^
vim +107 net/smc/smc_loopback.c
76
77 static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
78 void *client_priv)
79 {
80 struct smc_lo_dmb_node *dmb_node, *tmp_node;
81 struct smc_lo_dev *ldev = smcd->priv;
82 int sba_idx, rc;
83
84 /* check space for new dmb */
85 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LODEV_MAX_DMBS) {
86 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
87 break;
88 }
89 if (sba_idx == SMC_LODEV_MAX_DMBS)
90 return -ENOSPC;
91
92 dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
93 if (!dmb_node) {
94 rc = -ENOMEM;
95 goto err_bit;
96 }
97
98 dmb_node->sba_idx = sba_idx;
99 dmb_node->cpu_addr = kzalloc(dmb->dmb_len, GFP_KERNEL |
100 __GFP_NOWARN | __GFP_NORETRY |
101 __GFP_NOMEMALLOC);
102 if (!dmb_node->cpu_addr) {
103 rc = -ENOMEM;
104 goto err_node;
105 }
106 dmb_node->len = dmb->dmb_len;
> 107 dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
108
109 again:
110 /* add new dmb into hash table */
111 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
112 write_lock(&ldev->dmb_ht_lock);
113 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
114 if (tmp_node->token == dmb_node->token) {
115 write_unlock(&ldev->dmb_ht_lock);
116 goto again;
117 }
118 }
119 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
120 write_unlock(&ldev->dmb_ht_lock);
121
122 dmb->sba_idx = dmb_node->sba_idx;
123 dmb->dmb_tok = dmb_node->token;
124 dmb->cpu_addr = dmb_node->cpu_addr;
125 dmb->dma_addr = dmb_node->dma_addr;
126 dmb->dmb_len = dmb_node->len;
127
128 return 0;
129
130 err_node:
131 kfree(dmb_node);
132 err_bit:
133 clear_bit(sba_idx, ldev->sba_idx_mask);
134 return rc;
135 }
136
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On 2023/9/23 23:24, kernel test robot wrote:
> Hi Wen,
>
> kernel test robot noticed the following build warnings:
>
> [auto build test WARNING on net-next/main]
>
> url: https://github.com/intel-lab-lkp/linux/commits/Wen-Gu/net-smc-decouple-ism_dev-from-SMC-D-device-dump/20230920-010019
> base: net-next/main
> patch link: https://lore.kernel.org/r/1695134522-126655-13-git-send-email-guwen%40linux.alibaba.com
> patch subject: [PATCH net-next 12/18] net/smc: implement DMB-related operations of loopback
> config: arc-allmodconfig (https://download.01.org/0day-ci/archive/20230923/[email protected]/config)
> compiler: arceb-elf-gcc (GCC) 13.2.0
> reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230923/[email protected]/reproduce)
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <[email protected]>
> | Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
>
> All warnings (new ones prefixed by >>):
>
> net/smc/smc_loopback.c: In function 'smc_lo_register_dmb':
>>> net/smc/smc_loopback.c:107:30: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
> 107 | dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
> | ^
>
>
Will fix it on v4. Thanks!
> vim +107 net/smc/smc_loopback.c
>
> 76
> 77 static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
> 78 void *client_priv)
> 79 {
> 80 struct smc_lo_dmb_node *dmb_node, *tmp_node;
> 81 struct smc_lo_dev *ldev = smcd->priv;
> 82 int sba_idx, rc;
> 83
> 84 /* check space for new dmb */
> 85 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LODEV_MAX_DMBS) {
> 86 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
> 87 break;
> 88 }
> 89 if (sba_idx == SMC_LODEV_MAX_DMBS)
> 90 return -ENOSPC;
> 91
> 92 dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
> 93 if (!dmb_node) {
> 94 rc = -ENOMEM;
> 95 goto err_bit;
> 96 }
> 97
> 98 dmb_node->sba_idx = sba_idx;
> 99 dmb_node->cpu_addr = kzalloc(dmb->dmb_len, GFP_KERNEL |
> 100 __GFP_NOWARN | __GFP_NORETRY |
> 101 __GFP_NOMEMALLOC);
> 102 if (!dmb_node->cpu_addr) {
> 103 rc = -ENOMEM;
> 104 goto err_node;
> 105 }
> 106 dmb_node->len = dmb->dmb_len;
> > 107 dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
> 108
> 109 again:
> 110 /* add new dmb into hash table */
> 111 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
> 112 write_lock(&ldev->dmb_ht_lock);
> 113 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
> 114 if (tmp_node->token == dmb_node->token) {
> 115 write_unlock(&ldev->dmb_ht_lock);
> 116 goto again;
> 117 }
> 118 }
> 119 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
> 120 write_unlock(&ldev->dmb_ht_lock);
> 121
> 122 dmb->sba_idx = dmb_node->sba_idx;
> 123 dmb->dmb_tok = dmb_node->token;
> 124 dmb->cpu_addr = dmb_node->cpu_addr;
> 125 dmb->dma_addr = dmb_node->dma_addr;
> 126 dmb->dmb_len = dmb_node->len;
> 127
> 128 return 0;
> 129
> 130 err_node:
> 131 kfree(dmb_node);
> 132 err_bit:
> 133 clear_bit(sba_idx, ldev->sba_idx_mask);
> 134 return rc;
> 135 }
> 136
>