LU-16850 socklnd: remove ksnr_myiface from ksock_conn_cb

Drop ksnr_myiface: it is no longer needed since socklnd
TCP bonding got removed. There's one interface per
connection cb per peer_ni, and it can be accessed as
net->ksnn_interface.ksni_index.

Fix setting of ksni_nroutes accordingly. Duplication of
interface index in conn_cb and ksnn_interface was causing
the assertion
	ASSERTION( net->ksnn_interface.ksni_nroutes == 0 )
in ksocknal_shutdown() to fail if the corresponding
device is deregistered before lnd shutdown.

Modify test_214 of sanity-lnet to create connections so that
the scenario of socklnd shutdown with NI on a deregistered
interface is covered.

Lustre-change: https://review.whamcloud.com/51148
Lustre-commit: f6be07c457385cfacd9b802e4cade9f6f6ab7d6f

Fixes: a7ee03d7ca4185 ("LU-16378 lnet: handles unregister/register events)
Test-Parameters: trivial
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: I4de164c9e64aa770164a8320b9460fadce49aa06
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51326
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Cyril Bordage <cbordage@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c
index 3f222c2..76dbb72 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.c
+++ b/lnet/klnds/socklnd/socklnd.c
@@ -44,20 +44,6 @@
 static const struct lnet_lnd the_ksocklnd;
 struct ksock_nal_data ksocknal_data;
 
-static struct ksock_interface *
-ksocknal_index2iface(struct lnet_ni *ni, int index)
-{
-       struct ksock_net *net = ni->ni_data;
-       struct ksock_interface *iface;
-
-       iface = &net->ksnn_interface;
-
-       if (iface->ksni_index == index)
-               return iface;
-
-       return NULL;
-}
-
 static int ksocknal_ip2index(__u32 ipaddress, struct lnet_ni *ni)
 {
        struct net_device *dev;
@@ -106,7 +92,6 @@ ksocknal_create_conn_cb(__u32 ipaddr, int port)
        conn_cb->ksnr_retry_interval = 0;         /* OK to connect at any time */
        conn_cb->ksnr_ipaddr = ipaddr;
        conn_cb->ksnr_port = port;
-       conn_cb->ksnr_myiface = -1;
        conn_cb->ksnr_scheduled = 0;
        conn_cb->ksnr_connecting = 0;
        conn_cb->ksnr_connected = 0;
@@ -303,6 +288,7 @@ ksocknal_get_peer_info(struct lnet_ni *ni, int index,
        struct ksock_conn_cb *conn_cb;
        int i;
        int rc = -ENOENT;
+       struct ksock_net *net;
 
        read_lock(&ksocknal_data.ksnd_global_lock);
 
@@ -325,7 +311,9 @@ ksocknal_get_peer_info(struct lnet_ni *ni, int index,
                } else {
                        *id = peer_ni->ksnp_id;
                        ksocknal_dump_peer_debug_info(peer_ni);
-                       rc = choose_ipv4_src(myip, conn_cb->ksnr_myiface,
+                       net = ni->ni_data;
+                       rc = choose_ipv4_src(myip,
+                                            net->ksnn_interface.ksni_index,
                                             conn_cb->ksnr_ipaddr,
                                             ni->ni_net_ns);
                        *peer_ip = conn_cb->ksnr_ipaddr;
@@ -461,42 +449,10 @@ static void
 ksocknal_associate_cb_conn_locked(struct ksock_conn_cb *conn_cb,
                                  struct ksock_conn *conn)
 {
-       struct ksock_peer_ni *peer_ni = conn_cb->ksnr_peer;
        int type = conn->ksnc_type;
-       struct ksock_interface *iface;
-       int conn_iface = ksocknal_ip2index(conn->ksnc_myipaddr,
-                                          conn_cb->ksnr_peer->ksnp_ni);
 
        conn->ksnc_conn_cb = conn_cb;
        ksocknal_conn_cb_addref(conn_cb);
-
-       if (conn_cb->ksnr_myiface != conn_iface) {
-               if (conn_cb->ksnr_myiface < 0) {
-                       /* route wasn't bound locally yet (the initial route) */
-                       CDEBUG(D_NET, "Binding %s %pI4h to interface %d\n",
-                              libcfs_id2str(peer_ni->ksnp_id),
-                              &conn_cb->ksnr_ipaddr,
-                              conn_iface);
-               } else {
-                       CDEBUG(D_NET,
-                              "Rebinding %s %pI4h from interface %d to %d\n",
-                              libcfs_id2str(peer_ni->ksnp_id),
-                              &conn_cb->ksnr_ipaddr,
-                              conn_cb->ksnr_myiface,
-                              conn_iface);
-
-                       iface = ksocknal_index2iface(peer_ni->ksnp_ni,
-                                                    conn_cb->ksnr_myiface);
-                       if (iface)
-                               iface->ksni_nroutes--;
-               }
-               conn_cb->ksnr_myiface = conn_iface;
-               iface = ksocknal_index2iface(peer_ni->ksnp_ni,
-                                            conn_cb->ksnr_myiface);
-               if (iface)
-                       iface->ksni_nroutes++;
-       }
-
        ksocknal_incr_conn_count(conn_cb, type);
 
        /* Successful connection => further attempts can
@@ -522,19 +478,16 @@ ksocknal_add_conn_cb_locked(struct ksock_peer_ni *peer_ni,
        conn_cb->ksnr_peer = peer_ni;
        ksocknal_peer_addref(peer_ni);
 
-       /* set the conn_cb's interface to the current net's interface */
-       conn_cb->ksnr_myiface = net->ksnn_interface.ksni_index;
-       net->ksnn_interface.ksni_nroutes++;
-
        /* peer_ni's route list takes over my ref on 'route' */
        peer_ni->ksnp_conn_cb = conn_cb;
+       net->ksnn_interface.ksni_nroutes++;
 
        list_for_each(tmp, &peer_ni->ksnp_conns) {
                conn = list_entry(tmp, struct ksock_conn, ksnc_list);
 
                if (conn->ksnc_ipaddr != conn_cb->ksnr_ipaddr)
                        continue;
-
+               CDEBUG(D_NET, "call ksocknal_associate_cb_conn_locked\n");
                ksocknal_associate_cb_conn_locked(conn_cb, conn);
                /* keep going (typed conns) */
        }
@@ -544,9 +497,9 @@ static void
 ksocknal_del_conn_cb_locked(struct ksock_conn_cb *conn_cb)
 {
        struct ksock_peer_ni *peer_ni = conn_cb->ksnr_peer;
-       struct ksock_interface *iface;
        struct ksock_conn *conn;
        struct ksock_conn *cnxt;
+       struct ksock_net *net;
 
        LASSERT(!conn_cb->ksnr_deleted);
 
@@ -558,12 +511,9 @@ ksocknal_del_conn_cb_locked(struct ksock_conn_cb *conn_cb)
                ksocknal_close_conn_locked(conn, 0);
        }
 
-       if (conn_cb->ksnr_myiface >= 0) {
-               iface = ksocknal_index2iface(peer_ni->ksnp_ni,
-                                            conn_cb->ksnr_myiface);
-               if (iface)
-                       iface->ksni_nroutes--;
-       }
+       net = (struct ksock_net *)(peer_ni->ksnp_ni->ni_data);
+       net->ksnn_interface.ksni_nroutes--;
+       LASSERT(net->ksnn_interface.ksni_nroutes >= 0);
 
        conn_cb->ksnr_deleted = 1;
        ksocknal_conn_cb_decref(conn_cb);               /* drop peer_ni's ref */

diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h
index fefa0a8..c7fbfb4 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.h
+++ b/lnet/klnds/socklnd/socklnd.h
@@ -375,7 +375,6 @@ struct ksock_conn_cb {
                                                 * can happen next
                                                 */
        time64_t                ksnr_retry_interval;/* secs between retries */
-       int                     ksnr_myiface;   /* interface index */
        __u32                   ksnr_ipaddr;    /* IP address to connect to */
        int                     ksnr_port;      /* port to connect to */
        unsigned int            ksnr_scheduled:1;/* scheduled for attention */

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index 002b604..4180f89 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -1904,6 +1904,7 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb)
        time64_t deadline;
         int               retry_later = 0;
         int               rc = 0;
+       struct ksock_net *net;
 
        deadline = ktime_get_seconds() + ksocknal_timeout();
 
@@ -1959,8 +1960,9 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb)
                        goto failed;
                }
 
+               net = (struct ksock_net *)(peer_ni->ksnp_ni->ni_data);
                sock = lnet_connect(peer_ni->ksnp_id.nid,
-                                   conn_cb->ksnr_myiface,
+                                   net->ksnn_interface.ksni_index,
                                    conn_cb->ksnr_ipaddr, conn_cb->ksnr_port,
                                    peer_ni->ksnp_ni->ni_net_ns);
                if (IS_ERR(sock)) {

diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh
index ac32638..8abb3fe 100755 (executable)
--- a/lustre/tests/sanity-lnet.sh
+++ b/lustre/tests/sanity-lnet.sh
@@ -1879,6 +1879,9 @@ test_214() {
        check_ni_status "$nid1" up
        check_ni_status "$nid2" up
 
+       do_lnetctl ping --source $nid2 $nid1 ||
+               error "$LNETCTL ping --source $nid2 $nid1 failed"
+
        echo "Set $FAKE_IF down"
        echo "ip link set dev $FAKE_IF down"
        ip link set dev $FAKE_IF down