LU-10391 lnet: enhance connect/accept to support large addr

This patch introduces a version-2 of the acceptor protocol.  This
version uses a 'struct lnet_nid' rather than 'lnet_nid_t'

lnet_connect() now accepts a struct lnet_nid and uses version 2 if
necessary.  lnet_accept() accepts either v1 or v2.

Test-Parameters: trivial
Test-Parameters: serverversion=2.12 serverdistro=el7.9 testlist=runtests
Test-Parameters: clientversion=2.12 testlist=runtests
Signed-off-by: Mr NeilBrown <neilb@suse.de>
Change-Id: I523be0d217b6239c9791ff4fa536b9255c029ae7
Reviewed-on: https://review.whamcloud.com/42105
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 8fea3c7..97ad91e 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -529,6 +529,7 @@ extern struct lnet_ni *lnet_nid2ni_locked(lnet_nid_t nid, int cpt);
 extern struct lnet_ni *lnet_nid2ni_addref(lnet_nid_t nid);
 extern struct lnet_ni *lnet_net2ni_locked(__u32 net, int cpt);
 extern struct lnet_ni *lnet_net2ni_addref(__u32 net);
 extern struct lnet_ni *lnet_nid2ni_addref(lnet_nid_t nid);
 extern struct lnet_ni *lnet_net2ni_locked(__u32 net, int cpt);
 extern struct lnet_ni *lnet_net2ni_addref(__u32 net);

+extern struct lnet_ni *lnet_nid_to_ni_addref(struct lnet_nid *nid);

 struct lnet_net *lnet_get_net_locked(__u32 net_id);
 
 int lnet_lib_init(void);
 struct lnet_net *lnet_get_net_locked(__u32 net_id);
 
 int lnet_lib_init(void);


@@ -815,9 +816,9 @@ unsigned int lnet_get_lnd_timeout(void);
 void lnet_register_lnd(const struct lnet_lnd *lnd);
 void lnet_unregister_lnd(const struct lnet_lnd *lnd);
 
 void lnet_register_lnd(const struct lnet_lnd *lnd);
 void lnet_unregister_lnd(const struct lnet_lnd *lnd);
 

-struct socket *lnet_connect(lnet_nid_t peer_nid, int interface,
+struct socket *lnet_connect(struct lnet_nid *peer_nid, int interface,

                            struct sockaddr *peeraddr, struct net *ns);
                            struct sockaddr *peeraddr, struct net *ns);

-void lnet_connect_console_error(int rc, lnet_nid_t peer_nid,
+void lnet_connect_console_error(int rc, struct lnet_nid *peer_nid,

                                struct sockaddr *sa);
 int lnet_count_acceptor_nets(void);
 int lnet_acceptor_timeout(void);
                                struct sockaddr *sa);
 int lnet_count_acceptor_nets(void);
 int lnet_acceptor_timeout(void);

diff --git a/lnet/include/uapi/linux/lnet/lnet-idl.h b/lnet/include/uapi/linux/lnet/lnet-idl.h
index 303736a..10a9cb4 100644 (file)
--- a/lnet/include/uapi/linux/lnet/lnet-idl.h
+++ b/lnet/include/uapi/linux/lnet/lnet-idl.h
@@ -190,13 +190,22 @@ struct lnet_magicversion {
 
 /* Acceptor connection request */
 struct lnet_acceptor_connreq {
 
 /* Acceptor connection request */
 struct lnet_acceptor_connreq {

-       __u32   acr_magic;      /* PTL_ACCEPTOR_PROTO_MAGIC */
+       __u32   acr_magic;      /* LNET_PROTO_ACCEPTOR_MAGIC */

        __u32   acr_version;    /* protocol version */
        __u64   acr_nid;        /* target NID */
 } __attribute__((packed));
 
 #define LNET_PROTO_ACCEPTOR_VERSION    1
 
        __u32   acr_version;    /* protocol version */
        __u64   acr_nid;        /* target NID */
 } __attribute__((packed));
 
 #define LNET_PROTO_ACCEPTOR_VERSION    1
 

+struct lnet_acceptor_connreq_v2 {
+       __u32                   acr_magic;      /* LNET_PROTO_ACCEPTOR_MAGIC */
+       __u32                   acr_version;    /* protocol version - 2 */
+       struct lnet_nid         acr_nid;        /* target NID */
+} __attribute__((packed));
+
+/* For use with 16-byte addresses */
+#define LNET_PROTO_ACCEPTOR_VERSION_16  2
+

 struct lnet_counters_common {
        __u32   lcc_msgs_alloc;
        __u32   lcc_msgs_max;
 struct lnet_counters_common {
        __u32   lcc_msgs_alloc;
        __u32   lcc_msgs_max;

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index ec33332..fcdbd16 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -1961,12 +1961,12 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb)
                if (ktime_get_seconds() >= deadline) {
                        rc = -ETIMEDOUT;
                        lnet_connect_console_error(
                if (ktime_get_seconds() >= deadline) {
                        rc = -ETIMEDOUT;
                        lnet_connect_console_error(

-                               rc, lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+                               rc, &peer_ni->ksnp_id.nid,

                                (struct sockaddr *)&conn_cb->ksnr_addr);
                        goto failed;
                }
 
                                (struct sockaddr *)&conn_cb->ksnr_addr);
                        goto failed;
                }
 

-               sock = lnet_connect(lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+               sock = lnet_connect(&peer_ni->ksnp_id.nid,

                                    conn_cb->ksnr_myiface,
                                    (struct sockaddr *)&conn_cb->ksnr_addr,
                                    peer_ni->ksnp_ni->ni_net_ns);
                                    conn_cb->ksnr_myiface,
                                    (struct sockaddr *)&conn_cb->ksnr_addr,
                                    peer_ni->ksnp_ni->ni_net_ns);


@@ -1979,7 +1979,7 @@ ksocknal_connect(struct ksock_conn_cb *conn_cb)
                                          type);
                if (rc < 0) {
                        lnet_connect_console_error(
                                          type);
                if (rc < 0) {
                        lnet_connect_console_error(

-                               rc, lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+                               rc, &peer_ni->ksnp_id.nid,

                                (struct sockaddr *)&conn_cb->ksnr_addr);
                        goto failed;
                }
                                (struct sockaddr *)&conn_cb->ksnr_addr);
                        goto failed;
                }

diff --git a/lnet/lnet/acceptor.c b/lnet/lnet/acceptor.c
index 28c4ec7..51ffd29 100644 (file)
--- a/lnet/lnet/acceptor.c
+++ b/lnet/lnet/acceptor.c
@@ -90,57 +90,61 @@ lnet_acceptor_timeout(void)
 EXPORT_SYMBOL(lnet_acceptor_timeout);
 
 void
 EXPORT_SYMBOL(lnet_acceptor_timeout);
 
 void

-lnet_connect_console_error (int rc, lnet_nid_t peer_nid,
-                           struct sockaddr *sa)
+lnet_connect_console_error(int rc, struct lnet_nid *peer_nid,
+                          struct sockaddr *sa)

 {
        switch (rc) {
        /* "normal" errors */
        case -ECONNREFUSED:
                CNETERR("Connection to %s at host %pISp was refused: check that Lustre is running on that node.\n",
 {
        switch (rc) {
        /* "normal" errors */
        case -ECONNREFUSED:
                CNETERR("Connection to %s at host %pISp was refused: check that Lustre is running on that node.\n",

-                       libcfs_nid2str(peer_nid), sa);
+                       libcfs_nidstr(peer_nid), sa);

                break;
        case -EHOSTUNREACH:
        case -ENETUNREACH:
                CNETERR("Connection to %s at host %pIS was unreachable: the network or that node may be down, or Lustre may be misconfigured.\n",
                break;
        case -EHOSTUNREACH:
        case -ENETUNREACH:
                CNETERR("Connection to %s at host %pIS was unreachable: the network or that node may be down, or Lustre may be misconfigured.\n",

-                       libcfs_nid2str(peer_nid), sa);
+                       libcfs_nidstr(peer_nid), sa);

                break;
        case -ETIMEDOUT:
                CNETERR("Connection to %s at host %pISp took too long: that node may be hung or experiencing high load.\n",
                break;
        case -ETIMEDOUT:
                CNETERR("Connection to %s at host %pISp took too long: that node may be hung or experiencing high load.\n",

-                       libcfs_nid2str(peer_nid), sa);
+                       libcfs_nidstr(peer_nid), sa);

                break;
        case -ECONNRESET:
                LCONSOLE_ERROR_MSG(0x11b,
                                   "Connection to %s at host %pISp was reset: is it running a compatible version of Lustre and is %s one of its NIDs?\n",
                break;
        case -ECONNRESET:
                LCONSOLE_ERROR_MSG(0x11b,
                                   "Connection to %s at host %pISp was reset: is it running a compatible version of Lustre and is %s one of its NIDs?\n",

-                                  libcfs_nid2str(peer_nid), sa,
-                                  libcfs_nid2str(peer_nid));
+                                  libcfs_nidstr(peer_nid), sa,
+                                  libcfs_nidstr(peer_nid));

                break;
        case -EPROTO:
                LCONSOLE_ERROR_MSG(0x11c,
                                   "Protocol error connecting to %s at host %pISp: is it running a compatible version of Lustre?\n",
                break;
        case -EPROTO:
                LCONSOLE_ERROR_MSG(0x11c,
                                   "Protocol error connecting to %s at host %pISp: is it running a compatible version of Lustre?\n",

-                                  libcfs_nid2str(peer_nid), sa);
+                                  libcfs_nidstr(peer_nid), sa);

                break;
        case -EADDRINUSE:
                LCONSOLE_ERROR_MSG(0x11d,
                                   "No privileged ports available to connect to %s at host %pISp\n",
                break;
        case -EADDRINUSE:
                LCONSOLE_ERROR_MSG(0x11d,
                                   "No privileged ports available to connect to %s at host %pISp\n",

-                                  libcfs_nid2str(peer_nid), sa);
+                                  libcfs_nidstr(peer_nid), sa);

                break;
        default:
                LCONSOLE_ERROR_MSG(0x11e,
                                   "Unexpected error %d connecting to %s at host %pISp\n",
                break;
        default:
                LCONSOLE_ERROR_MSG(0x11e,
                                   "Unexpected error %d connecting to %s at host %pISp\n",

-                                  rc, libcfs_nid2str(peer_nid), sa);
+                                  rc, libcfs_nidstr(peer_nid), sa);

                break;
        }
 }
 EXPORT_SYMBOL(lnet_connect_console_error);
 
 struct socket *
                break;
        }
 }
 EXPORT_SYMBOL(lnet_connect_console_error);
 
 struct socket *

-lnet_connect(lnet_nid_t peer_nid, int interface, struct sockaddr *peeraddr,
+lnet_connect(struct lnet_nid *peer_nid, int interface,
+            struct sockaddr *peeraddr,

             struct net *ns)
 {
             struct net *ns)
 {

-       struct lnet_acceptor_connreq cr;
-       struct socket           *sock;
-       int                     rc;
-       int                     port;
+       struct lnet_acceptor_connreq cr1;
+       struct lnet_acceptor_connreq_v2 cr2;
+       void *cr;
+       int crsize;
+       struct socket *sock;
+       int rc;
+       int port;

 
        BUILD_BUG_ON(sizeof(cr) > 16); /* not too big to be on the stack */
 
 
        BUILD_BUG_ON(sizeof(cr) > 16); /* not too big to be on the stack */
 


@@ -161,20 +165,32 @@ lnet_connect(lnet_nid_t peer_nid, int interface, struct sockaddr *peeraddr,
 
                BUILD_BUG_ON(LNET_PROTO_ACCEPTOR_VERSION != 1);
 
 
                BUILD_BUG_ON(LNET_PROTO_ACCEPTOR_VERSION != 1);
 

-               cr.acr_magic   = LNET_PROTO_ACCEPTOR_MAGIC;
-               cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
-               cr.acr_nid     = peer_nid;
-
-               if (the_lnet.ln_testprotocompat) {
-                       /* single-shot proto check */
-                       if (test_and_clear_bit(2, &the_lnet.ln_testprotocompat))
-                               cr.acr_version++;
-                       if (test_and_clear_bit(3, &the_lnet.ln_testprotocompat))
-                               cr.acr_magic = LNET_PROTO_MAGIC;
+               if (nid_is_nid4(peer_nid)) {
+                       cr1.acr_magic   = LNET_PROTO_ACCEPTOR_MAGIC;
+                       cr1.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
+                       cr1.acr_nid     = lnet_nid_to_nid4(peer_nid);
+                       cr = &cr1;
+                       crsize = sizeof(cr1);
+
+                       if (the_lnet.ln_testprotocompat) {
+                               /* single-shot proto check */
+                               if (test_and_clear_bit(
+                                           2, &the_lnet.ln_testprotocompat))
+                                       cr1.acr_version++;
+                               if (test_and_clear_bit(
+                                           3, &the_lnet.ln_testprotocompat))
+                                       cr1.acr_magic = LNET_PROTO_MAGIC;
+                       }
+
+               } else {
+                       cr2.acr_magic   = LNET_PROTO_ACCEPTOR_MAGIC;
+                       cr2.acr_version = LNET_PROTO_ACCEPTOR_VERSION_16;
+                       cr2.acr_nid     = *peer_nid;
+                       cr = &cr2;
+                       crsize = sizeof(cr2);

                }
 
                }
 

-               rc = lnet_sock_write(sock, &cr, sizeof(cr),
-                                      accept_timeout);
+               rc = lnet_sock_write(sock, cr, crsize, accept_timeout);

                if (rc != 0)
                        goto failed_sock;
 
                if (rc != 0)
                        goto failed_sock;
 


@@ -196,11 +212,14 @@ static int
 lnet_accept(struct socket *sock, __u32 magic)
 {
        struct lnet_acceptor_connreq cr;
 lnet_accept(struct socket *sock, __u32 magic)
 {
        struct lnet_acceptor_connreq cr;

+       struct lnet_acceptor_connreq_v2 cr2;
+       struct lnet_nid nid;

        struct sockaddr_storage peer;
        struct sockaddr_storage peer;

-       int                     rc;
-       int                     flip;
+       int peer_version;
+       int rc;
+       int flip;

        struct lnet_ni *ni;
        struct lnet_ni *ni;

-       char                   *str;
+       char *str;

 
        LASSERT(sizeof(cr) <= 16);              /* not too big for the stack */
 
 
        LASSERT(sizeof(cr) <= 16);              /* not too big for the stack */
 


@@ -255,12 +274,13 @@ lnet_accept(struct socket *sock, __u32 magic)
        if (flip)
                __swab32s(&cr.acr_version);
 
        if (flip)
                __swab32s(&cr.acr_version);
 

-       if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION) {
+       switch (cr.acr_version) {
+       default:

                /* future version compatibility!
                 * An acceptor-specific protocol rev will first send a version
                 * query.  I send back my current version to tell her I'm
                 * "old". */
                /* future version compatibility!
                 * An acceptor-specific protocol rev will first send a version
                 * query.  I send back my current version to tell her I'm
                 * "old". */

-               int peer_version = cr.acr_version;
+               peer_version = cr.acr_version;

 
                memset(&cr, 0, sizeof(cr));
                cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
 
                memset(&cr, 0, sizeof(cr));
                cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;


@@ -273,30 +293,48 @@ lnet_accept(struct socket *sock, __u32 magic)
                        CERROR("Error sending magic+version in response to version %d from %pIS: %d\n",
                               peer_version, &peer, rc);
                return -EPROTO;
                        CERROR("Error sending magic+version in response to version %d from %pIS: %d\n",
                               peer_version, &peer, rc);
                return -EPROTO;

-       }

 
 

-       rc = lnet_sock_read(sock, &cr.acr_nid,
-                             sizeof(cr) -
-                             offsetof(struct lnet_acceptor_connreq, acr_nid),
-                             accept_timeout);
+       case LNET_PROTO_ACCEPTOR_VERSION:
+
+               rc = lnet_sock_read(sock, &cr.acr_nid,
+                                   sizeof(cr) -
+                                   offsetof(struct lnet_acceptor_connreq,
+                                            acr_nid),
+                                   accept_timeout);
+               if (rc)
+                       break;
+               if (flip)
+                       __swab64s(&cr.acr_nid);
+
+               lnet_nid4_to_nid(cr.acr_nid, &nid);
+               break;
+
+       case LNET_PROTO_ACCEPTOR_VERSION_16:
+               rc = lnet_sock_read(sock, &cr2.acr_nid,
+                                   sizeof(cr2) -
+                                   offsetof(struct lnet_acceptor_connreq_v2,
+                                            acr_nid),
+                                   accept_timeout);
+               if (rc)
+                       break;
+               nid = cr2.acr_nid;
+               break;
+       }

        if (rc != 0) {
                CERROR("Error %d reading connection request from %pIS\n",
                       rc, &peer);
                return -EIO;
        }
 
        if (rc != 0) {
                CERROR("Error %d reading connection request from %pIS\n",
                       rc, &peer);
                return -EIO;
        }
 

-       if (flip)
-               __swab64s(&cr.acr_nid);
-
-       ni = lnet_nid2ni_addref(cr.acr_nid);
+       ni = lnet_nid_to_ni_addref(&nid);

        if (ni == NULL ||               /* no matching net */
        if (ni == NULL ||               /* no matching net */

-           lnet_nid_to_nid4(&ni->ni_nid) != cr.acr_nid) {
+           !nid_same(&ni->ni_nid, &nid)) {

                /* right NET, wrong NID! */
                if (ni != NULL)
                        lnet_ni_decref(ni);
                LCONSOLE_ERROR_MSG(0x120,
                                   "Refusing connection from %pIS for %s: No matching NI\n",
                /* right NET, wrong NID! */
                if (ni != NULL)
                        lnet_ni_decref(ni);
                LCONSOLE_ERROR_MSG(0x120,
                                   "Refusing connection from %pIS for %s: No matching NI\n",

-                                  &peer, libcfs_nid2str(cr.acr_nid));
+                                  &peer, libcfs_nidstr(&nid));

                return -EPERM;
        }
 
                return -EPERM;
        }
 


@@ -305,12 +343,11 @@ lnet_accept(struct socket *sock, __u32 magic)
                lnet_ni_decref(ni);
                LCONSOLE_ERROR_MSG(0x121,
                                   "Refusing connection from %pIS for %s: NI doesn not accept IP connections\n",
                lnet_ni_decref(ni);
                LCONSOLE_ERROR_MSG(0x121,
                                   "Refusing connection from %pIS for %s: NI doesn not accept IP connections\n",

-                                 &peer, libcfs_nid2str(cr.acr_nid));
+                                 &peer, libcfs_nidstr(&nid));

                return -EPERM;
        }
 
                return -EPERM;
        }
 

-       CDEBUG(D_NET, "Accept %s from %pI4h\n",
-              libcfs_nid2str(cr.acr_nid), &peer);
+       CDEBUG(D_NET, "Accept %s from %pI4h\n", libcfs_nidstr(&nid), &peer);

 
        rc = ni->ni_net->net_lnd->lnd_accept(ni, sock);
 
 
        rc = ni->ni_net->net_lnd->lnd_accept(ni, sock);
 

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 9908917..45abed8 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -883,6 +883,15 @@ static void lnet_assert_wire_constants(void)
        BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_nid) != 8);
        BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_nid) != 8);
 
        BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_nid) != 8);
        BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_nid) != 8);
 

+       /* Checks for struct lnet_acceptor_connreq_v2 */
+       BUILD_BUG_ON((int)sizeof(struct lnet_acceptor_connreq_v2) != 28);
+       BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_magic) != 0);
+       BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_magic) != 4);
+       BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_version) != 4);
+       BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_version) != 4);
+       BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_nid) != 8);
+       BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_nid) != 20);
+

        /* Checks for struct lnet_counters_common */
        BUILD_BUG_ON((int)sizeof(struct lnet_counters_common) != 60);
        BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_msgs_alloc) != 0);
        /* Checks for struct lnet_counters_common */
        BUILD_BUG_ON((int)sizeof(struct lnet_counters_common) != 60);
        BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_msgs_alloc) != 0);