From 12615ae3456fa932fa90c888e166aadd7cfb1f58 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Fri, 26 May 2023 10:42:23 -0700 Subject: [PATCH] LU-16850 socklnd: remove ksnr_myiface from ksock_conn_cb Drop ksnr_myiface: it is no longer needed since socklnd TCP bonding got removed. There's one interface per connection cb per peer_ni, and it can be accessed as net->ksnn_interface.ksni_index. Fix setting of ksni_nroutes accordingly. Duplication of interface index in conn_cb and ksnn_interface was causing the assertion ASSERTION( net->ksnn_interface.ksni_nroutes == 0 ) in ksocknal_shutdown() to fail if the corresponding device is deregistered before lnd shutdown. Modify test_214 of sanity-lnet to create connections so that the scenario of socklnd shutdown with NI on a deregistered interface is covered. Lustre-change: https://review.whamcloud.com/51148 Lustre-commit: f6be07c457385cfacd9b802e4cade9f6f6ab7d6f Fixes: a7ee03d7ca4185 ("LU-16378 lnet: handles unregister/register events) Test-Parameters: trivial Signed-off-by: Serguei Smirnov Change-Id: I4de164c9e64aa770164a8320b9460fadce49aa06 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51326 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Frank Sehr Reviewed-by: Cyril Bordage Reviewed-by: Andreas Dilger --- lnet/klnds/socklnd/socklnd.c | 70 ++++++----------------------------------- lnet/klnds/socklnd/socklnd.h | 1 - lnet/klnds/socklnd/socklnd_cb.c | 4 ++- lustre/tests/sanity-lnet.sh | 3 ++ 4 files changed, 16 insertions(+), 62 deletions(-) diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 3f222c2..76dbb72 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -44,20 +44,6 @@ static const struct lnet_lnd the_ksocklnd; struct ksock_nal_data ksocknal_data; -static struct ksock_interface * -ksocknal_index2iface(struct lnet_ni *ni, int index) -{ - struct ksock_net *net = ni->ni_data; - struct ksock_interface *iface; - - iface = &net->ksnn_interface; - - if (iface->ksni_index == index) - return iface; - - return NULL; -} - static int ksocknal_ip2index(__u32 ipaddress, struct lnet_ni *ni) { struct net_device *dev; @@ -106,7 +92,6 @@ ksocknal_create_conn_cb(__u32 ipaddr, int port) conn_cb->ksnr_retry_interval = 0; /* OK to connect at any time */ conn_cb->ksnr_ipaddr = ipaddr; conn_cb->ksnr_port = port; - conn_cb->ksnr_myiface = -1; conn_cb->ksnr_scheduled = 0; conn_cb->ksnr_connecting = 0; conn_cb->ksnr_connected = 0; @@ -303,6 +288,7 @@ ksocknal_get_peer_info(struct lnet_ni *ni, int index, struct ksock_conn_cb *conn_cb; int i; int rc = -ENOENT; + struct ksock_net *net; read_lock(&ksocknal_data.ksnd_global_lock); @@ -325,7 +311,9 @@ ksocknal_get_peer_info(struct lnet_ni *ni, int index, } else { *id = peer_ni->ksnp_id; ksocknal_dump_peer_debug_info(peer_ni); - rc = choose_ipv4_src(myip, conn_cb->ksnr_myiface, + net = ni->ni_data; + rc = choose_ipv4_src(myip, + net->ksnn_interface.ksni_index, conn_cb->ksnr_ipaddr, ni->ni_net_ns); *peer_ip = conn_cb->ksnr_ipaddr; @@ -461,42 +449,10 @@ static void ksocknal_associate_cb_conn_locked(struct ksock_conn_cb *conn_cb, struct ksock_conn *conn) { - struct ksock_peer_ni *peer_ni = conn_cb->ksnr_peer; int type = conn->ksnc_type; - struct ksock_interface *iface; - int conn_iface = ksocknal_ip2index(conn->ksnc_myipaddr, - conn_cb->ksnr_peer->ksnp_ni); conn->ksnc_conn_cb = conn_cb; ksocknal_conn_cb_addref(conn_cb); - - if (conn_cb->ksnr_myiface != conn_iface) { - if (conn_cb->ksnr_myiface < 0) { - /* route wasn't bound locally yet (the initial route) */ - CDEBUG(D_NET, "Binding %s %pI4h to interface %d\n", - libcfs_id2str(peer_ni->ksnp_id), - &conn_cb->ksnr_ipaddr, - conn_iface); - } else { - CDEBUG(D_NET, - "Rebinding %s %pI4h from interface %d to %d\n", - libcfs_id2str(peer_ni->ksnp_id), - &conn_cb->ksnr_ipaddr, - conn_cb->ksnr_myiface, - conn_iface); - - iface = ksocknal_index2iface(peer_ni->ksnp_ni, - conn_cb->ksnr_myiface); - if (iface) - iface->ksni_nroutes--; - } - conn_cb->ksnr_myiface = conn_iface; - iface = ksocknal_index2iface(peer_ni->ksnp_ni, - conn_cb->ksnr_myiface); - if (iface) - iface->ksni_nroutes++; - } - ksocknal_incr_conn_count(conn_cb, type); /* Successful connection => further attempts can @@ -522,19 +478,16 @@ ksocknal_add_conn_cb_locked(struct ksock_peer_ni *peer_ni, conn_cb->ksnr_peer = peer_ni; ksocknal_peer_addref(peer_ni); - /* set the conn_cb's interface to the current net's interface */ - conn_cb->ksnr_myiface = net->ksnn_interface.ksni_index; - net->ksnn_interface.ksni_nroutes++; - /* peer_ni's route list takes over my ref on 'route' */ peer_ni->ksnp_conn_cb = conn_cb; + net->ksnn_interface.ksni_nroutes++; list_for_each(tmp, &peer_ni->ksnp_conns) { conn = list_entry(tmp, struct ksock_conn, ksnc_list); if (conn->ksnc_ipaddr != conn_cb->ksnr_ipaddr) continue; - + CDEBUG(D_NET, "call ksocknal_associate_cb_conn_locked\n"); ksocknal_associate_cb_conn_locked(conn_cb, conn); /* keep going (typed conns) */ } @@ -544,9 +497,9 @@ static void ksocknal_del_conn_cb_locked(struct ksock_conn_cb *conn_cb) { struct ksock_peer_ni *peer_ni = conn_cb->ksnr_peer; - struct ksock_interface *iface; struct ksock_conn *conn; struct ksock_conn *cnxt; + struct ksock_net *net; LASSERT(!conn_cb->ksnr_deleted); @@ -558,12 +511,9 @@ ksocknal_del_conn_cb_locked(struct ksock_conn_cb *conn_cb) ksocknal_close_conn_locked(conn, 0); } - if (conn_cb->ksnr_myiface >= 0) { - iface = ksocknal_index2iface(peer_ni->ksnp_ni, - conn_cb->ksnr_myiface); - if (iface) - iface->ksni_nroutes--; - } + net = (struct ksock_net *)(peer_ni->ksnp_ni->ni_data); + net->ksnn_interface.ksni_nroutes--; + LASSERT(net->ksnn_interface.ksni_nroutes >= 0); conn_cb->ksnr_deleted = 1; ksocknal_conn_cb_decref(conn_cb); /* drop peer_ni's ref */ diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index fefa0a8..c7fbfb4 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -375,7 +375,6 @@ struct ksock_conn_cb { * can happen next */ time64_t ksnr_retry_interval;/* secs between retries */ - int ksnr_myiface; /* interface index */ __u32 ksnr_ipaddr; /* IP address to connect to */ int ksnr_port; /* port to connect to */ unsigned int ksnr_scheduled:1;/* scheduled for attention */ diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 002b604..4180f89 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -1904,6 +1904,7 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb) time64_t deadline; int retry_later = 0; int rc = 0; + struct ksock_net *net; deadline = ktime_get_seconds() + ksocknal_timeout(); @@ -1959,8 +1960,9 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb) goto failed; } + net = (struct ksock_net *)(peer_ni->ksnp_ni->ni_data); sock = lnet_connect(peer_ni->ksnp_id.nid, - conn_cb->ksnr_myiface, + net->ksnn_interface.ksni_index, conn_cb->ksnr_ipaddr, conn_cb->ksnr_port, peer_ni->ksnp_ni->ni_net_ns); if (IS_ERR(sock)) { diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index ac32638..8abb3fe 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -1879,6 +1879,9 @@ test_214() { check_ni_status "$nid1" up check_ni_status "$nid2" up + do_lnetctl ping --source $nid2 $nid1 || + error "$LNETCTL ping --source $nid2 $nid1 failed" + echo "Set $FAKE_IF down" echo "ip link set dev $FAKE_IF down" ip link set dev $FAKE_IF down -- 1.8.3.1