LU-10391 lnet: track pinginfo size in bytes, not nis.

When we extend the pinginfo to be able to store large-address nids,
there could be nids of different sizes in it.  So using the number of
nis to track the size won't work.  So change to using the number of
bytes.  i.e.  the total size of the 'struct lnet_ping_info'.

This affects pb_nnis in the ping_buffer, and the global
ln_push_target_nnis.

LNET_PING_INFO_SIZE is removed as size won't depend on number of nids
any more.

When determining the number of bytes expected in a received ping_info,
use a new macro lnet_ping_info_size() which can extract information
as required from the ping_info.

Note that lnet_ping_target_create() now initializes pi_nis to 0.
Setting the initial size doesn't seem to be useful.

Test-Parameters: trivial testlist=sanity-lnet
Test-Parameters: serverversion=2.12 serverdistro=el7.9 testlist=runtests
Test-Parameters: clientversion=2.12 testlist=runtests
Signed-off-by: Mr NeilBrown <neilb@suse.de>
Change-Id: I7727b784ed9a7510959d5ec41f8df3851adb78ed
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/44627
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: jsimmons <jsimmons@infradead.org>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 0974c95..3398402 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -696,7 +696,7 @@ void lnet_prep_send(struct lnet_msg *msg, int type,
 int lnet_send(struct lnet_nid *nid, struct lnet_msg *msg,
              struct lnet_nid *rtr_nid);
 int lnet_send_ping(struct lnet_nid *dest_nid, struct lnet_handle_md *mdh,
-                  int nnis, void *user_ptr, lnet_handler_t handler,
+                  int bytes, void *user_ptr, lnet_handler_t handler,
                   bool recovery);
 void lnet_return_tx_credits_locked(struct lnet_msg *msg);
 void lnet_return_rx_credits_locked(struct lnet_msg *msg);
@@ -937,7 +937,7 @@ void lnet_wait_router_start(void);
 void lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf);
 
 int lnet_ping_info_validate(struct lnet_ping_info *pinfo);
-struct lnet_ping_buffer *lnet_ping_buffer_alloc(int nnis, gfp_t gfp);
+struct lnet_ping_buffer *lnet_ping_buffer_alloc(int bytes, gfp_t gfp);
 void lnet_ping_buffer_free(struct lnet_ping_buffer *pbuf);
 
 static inline void lnet_ping_buffer_addref(struct lnet_ping_buffer *pbuf)
@@ -955,7 +955,7 @@ static inline void lnet_ping_buffer_decref(struct lnet_ping_buffer *pbuf)
 
 static inline int lnet_push_target_resize_needed(void)
 {
-       return the_lnet.ln_push_target->pb_nnis < the_lnet.ln_push_target_nnis;
+       return the_lnet.ln_push_target->pb_nbytes < the_lnet.ln_push_target_nbytes;
 }
 
 int lnet_push_target_resize(void);

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index 1d95275..4b15d07 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -558,14 +558,14 @@ struct lnet_ni {
  * area that may be overwritten by network data.
  */
 struct lnet_ping_buffer {
-       int                     pb_nnis;
+       int                     pb_nbytes;      /* sizeof pb_info */
        atomic_t                pb_refcnt;
        bool                    pb_needs_post;
        struct lnet_ping_info   pb_info;
 };
 
-#define LNET_PING_BUFFER_SIZE(NNIDS) \
-       offsetof(struct lnet_ping_buffer, pb_info.pi_ni[NNIDS])
+#define LNET_PING_BUFFER_SIZE(bytes) \
+       (offsetof(struct lnet_ping_buffer, pb_info) + bytes)
 #define LNET_PING_BUFFER_LONI(PBUF)    ((PBUF)->pb_info.pi_ni[0].ns_nid)
 #define LNET_PING_BUFFER_SEQNO(PBUF)   ((PBUF)->pb_info.pi_ni[0].ns_status)
 
@@ -724,8 +724,8 @@ struct lnet_peer {
        /* MD handle for push in progress */
        struct lnet_handle_md   lp_push_mdh;
 
-       /* number of NIDs for sizing push data */
-       int                     lp_data_nnis;
+       /* number of bytes for sizing pb_info in push data */
+       int                     lp_data_bytes;
 
        /* NI config sequence number of peer */
        __u32                   lp_peer_seqno;
@@ -1210,7 +1210,8 @@ struct lnet {
        lnet_handler_t                  ln_push_target_handler;
        struct lnet_handle_md           ln_push_target_md;
        struct lnet_ping_buffer         *ln_push_target;
-       int                             ln_push_target_nnis;
+       /* bytes needed for pb_info to receive push */
+       int                             ln_push_target_nbytes;
 
        /* discovery event queue handle */
        lnet_handler_t                  ln_dc_handler;

diff --git a/lnet/include/uapi/linux/lnet/lnet-idl.h b/lnet/include/uapi/linux/lnet/lnet-idl.h
index d10b588..0e2b1f8 100644 (file)
--- a/lnet/include/uapi/linux/lnet/lnet-idl.h
+++ b/lnet/include/uapi/linux/lnet/lnet-idl.h
@@ -291,9 +291,13 @@ struct lnet_ping_info {
        struct lnet_ni_status   pi_ni[0];
 } __attribute__((packed));
 
-#define LNET_PING_INFO_SIZE(NNIDS) \
-       offsetof(struct lnet_ping_info, pi_ni[NNIDS])
+#define LNET_PING_INFO_HDR_SIZE \
+       offsetof(struct lnet_ping_info, pi_ni[0])
+#define LNET_PING_INFO_MIN_SIZE \
+       offsetof(struct lnet_ping_info, pi_ni[LNET_INTERFACES_MIN])
 #define LNET_PING_INFO_LONI(PINFO)      ((PINFO)->pi_ni[0].ns_nid)
 #define LNET_PING_INFO_SEQNO(PINFO)     ((PINFO)->pi_ni[0].ns_status)
+#define lnet_ping_info_size(pinfo)     \
+       offsetof(struct lnet_ping_info, pi_ni[(pinfo)->pi_nnis])
 
 #endif

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 0fc1583..036c035 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -1803,13 +1803,13 @@ lnet_count_acceptor_nets(void)
 }
 
 struct lnet_ping_buffer *
-lnet_ping_buffer_alloc(int nnis, gfp_t gfp)
+lnet_ping_buffer_alloc(int nbytes, gfp_t gfp)
 {
        struct lnet_ping_buffer *pbuf;
 
-       LIBCFS_ALLOC_GFP(pbuf, LNET_PING_BUFFER_SIZE(nnis), gfp);
+       LIBCFS_ALLOC_GFP(pbuf, LNET_PING_BUFFER_SIZE(nbytes), gfp);
        if (pbuf) {
-               pbuf->pb_nnis = nnis;
+               pbuf->pb_nbytes = nbytes;       /* sizeof of pb_info */
                pbuf->pb_needs_post = false;
                atomic_set(&pbuf->pb_refcnt, 1);
        }
@@ -1821,21 +1821,21 @@ void
 lnet_ping_buffer_free(struct lnet_ping_buffer *pbuf)
 {
        LASSERT(atomic_read(&pbuf->pb_refcnt) == 0);
-       LIBCFS_FREE(pbuf, LNET_PING_BUFFER_SIZE(pbuf->pb_nnis));
+       LIBCFS_FREE(pbuf, LNET_PING_BUFFER_SIZE(pbuf->pb_nbytes));
 }
 
 static struct lnet_ping_buffer *
-lnet_ping_target_create(int nnis)
+lnet_ping_target_create(int nbytes)
 {
        struct lnet_ping_buffer *pbuf;
 
-       pbuf = lnet_ping_buffer_alloc(nnis, GFP_NOFS);
+       pbuf = lnet_ping_buffer_alloc(nbytes, GFP_NOFS);
        if (pbuf == NULL) {
-               CERROR("Can't allocate ping source [%d]\n", nnis);
+               CERROR("Can't allocate ping source [%d]\n", nbytes);
                return NULL;
        }
 
-       pbuf->pb_info.pi_nnis = nnis;
+       pbuf->pb_info.pi_nnis = 0;
        pbuf->pb_info.pi_pid = the_lnet.ln_pid;
        pbuf->pb_info.pi_magic = LNET_PROTO_PING_MAGIC;
        pbuf->pb_info.pi_features =
@@ -1845,52 +1845,55 @@ lnet_ping_target_create(int nnis)
 }
 
 static inline int
-lnet_get_net_ni_count_locked(struct lnet_net *net)
+lnet_get_net_ni_bytes_locked(struct lnet_net *net)
 {
-       struct lnet_ni  *ni;
-       int             count = 0;
+       struct lnet_ni *ni;
+       int bytes = 0;
 
        list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
-               count++;
+               if (nid_is_nid4(&ni->ni_nid))
+                       bytes += sizeof(struct lnet_ni_status);
 
-       return count;
+       return bytes;
 }
 
 static inline int
-lnet_get_net_ni_count_pre(struct lnet_net *net)
+lnet_get_net_ni_bytes_pre(struct lnet_net *net)
 {
-       struct lnet_ni  *ni;
-       int             count = 0;
+       struct lnet_ni *ni;
+       int bytes = 0;
 
        list_for_each_entry(ni, &net->net_ni_added, ni_netlist)
-               count++;
+               if (nid_is_nid4(&ni->ni_nid))
+                       bytes += sizeof(struct lnet_ni_status);
 
-       return count;
+       return bytes;
 }
 
 static inline int
-lnet_get_ni_count(void)
+lnet_get_ni_bytes(void)
 {
-       struct lnet_ni  *ni;
+       struct lnet_ni *ni;
        struct lnet_net *net;
-       int             count = 0;
+       int bytes = 0;
 
        lnet_net_lock(0);
 
        list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
                list_for_each_entry(ni, &net->net_ni_list, ni_netlist)
-                       count++;
+                       if (nid_is_nid4(&ni->ni_nid))
+                               bytes += sizeof(struct lnet_ni_status);
        }
 
        lnet_net_unlock(0);
 
-       return count;
+       return bytes;
 }
 
 void
 lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf)
 {
-       struct lnet_ni_status *stat;
+       struct lnet_ni_status *stat, *end;
        int nnis;
        int i;
 
@@ -1899,10 +1902,9 @@ lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf)
        __swab32s(&pbuf->pb_info.pi_pid);
        __swab32s(&pbuf->pb_info.pi_nnis);
        nnis = pbuf->pb_info.pi_nnis;
-       if (nnis > pbuf->pb_nnis)
-               nnis = pbuf->pb_nnis;
-       for (i = 0; i < nnis; i++) {
-               stat = &pbuf->pb_info.pi_ni[i];
+       stat = &pbuf->pb_info.pi_ni[0];
+       end = (void *)&pbuf->pb_info + pbuf->pb_nbytes;
+       for (i = 0; i < nnis && stat + 1 <= end; i++, stat++) {
                __swab64s(&stat->ns_nid);
                __swab32s(&stat->ns_status);
        }
@@ -1959,7 +1961,7 @@ lnet_ping_target_event_handler(struct lnet_event *event)
 static int
 lnet_ping_target_setup(struct lnet_ping_buffer **ppbuf,
                       struct lnet_handle_md *ping_mdh,
-                      int ni_count, bool set_eq)
+                      int ni_bytes, bool set_eq)
 {
        struct lnet_processid id = {
                .nid = LNET_ANY_NID,
@@ -1973,7 +1975,7 @@ lnet_ping_target_setup(struct lnet_ping_buffer **ppbuf,
                the_lnet.ln_ping_target_handler =
                        lnet_ping_target_event_handler;
 
-       *ppbuf = lnet_ping_target_create(ni_count);
+       *ppbuf = lnet_ping_target_create(ni_bytes);
        if (*ppbuf == NULL) {
                rc = -ENOMEM;
                goto fail_free_eq;
@@ -1991,7 +1993,7 @@ lnet_ping_target_setup(struct lnet_ping_buffer **ppbuf,
 
        /* initialize md content */
        md.start     = &(*ppbuf)->pb_info;
-       md.length    = LNET_PING_INFO_SIZE((*ppbuf)->pb_nnis);
+       md.length    = (*ppbuf)->pb_nbytes;
        md.threshold = LNET_MD_THRESH_INF;
        md.max_size  = 0;
        md.options   = LNET_MD_OP_GET | LNET_MD_TRUNCATE |
@@ -2032,21 +2034,19 @@ lnet_ping_md_unlink(struct lnet_ping_buffer *pbuf,
 static void
 lnet_ping_target_install_locked(struct lnet_ping_buffer *pbuf)
 {
-       struct lnet_ni          *ni;
-       struct lnet_net         *net;
-       struct lnet_ni_status *ns;
-       int                     i;
-       int                     rc;
+       struct lnet_ni *ni;
+       struct lnet_net *net;
+       struct lnet_ni_status *ns, *end;
+       int rc;
 
-       i = 0;
+       pbuf->pb_info.pi_nnis = 0;
+       ns = &pbuf->pb_info.pi_ni[0];
+       end = (void *)&pbuf->pb_info + pbuf->pb_nbytes;
        list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
                list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
-                       LASSERT(i < pbuf->pb_nnis);
-
-                       ns = &pbuf->pb_info.pi_ni[i];
-
                        if (!nid_is_nid4(&ni->ni_nid))
                                continue;
+                       LASSERT(ns + 1 <= end);
                        ns->ns_nid = lnet_nid_to_nid4(&ni->ni_nid);
 
                        lnet_ni_lock(ni);
@@ -2054,11 +2054,12 @@ lnet_ping_target_install_locked(struct lnet_ping_buffer *pbuf)
                        ni->ni_status = &ns->ns_status;
                        lnet_ni_unlock(ni);
 
-                       i++;
+                       pbuf->pb_info.pi_nnis++;
+                       ns++;
                }
        }
-       /*
-        * We (ab)use the ns_status of the loopback interface to
+
+       /* We (ab)use the ns_status of the loopback interface to
         * transmit the sequence number. The first interface listed
         * must be the loopback interface.
         */
@@ -2127,13 +2128,13 @@ int lnet_push_target_resize(void)
        struct lnet_handle_md old_mdh;
        struct lnet_ping_buffer *pbuf;
        struct lnet_ping_buffer *old_pbuf;
-       int nnis;
+       int nbytes;
        int rc;
 
 again:
-       nnis = the_lnet.ln_push_target_nnis;
-       if (nnis <= 0) {
-               CDEBUG(D_NET, "Invalid nnis %d\n", nnis);
+       nbytes = the_lnet.ln_push_target_nbytes;
+       if (nbytes <= 0) {
+               CDEBUG(D_NET, "Invalid nbytes %d\n", nbytes);
                return -EINVAL;
        }
 
@@ -2141,9 +2142,9 @@ again:
         * dropped when we need to resize again (see "old_pbuf" below) or when
         * LNet is shutdown (see lnet_push_target_fini())
         */
-       pbuf = lnet_ping_buffer_alloc(nnis, GFP_NOFS);
+       pbuf = lnet_ping_buffer_alloc(nbytes, GFP_NOFS);
        if (!pbuf) {
-               CDEBUG(D_NET, "Can't allocate pbuf for nnis %d\n", nnis);
+               CDEBUG(D_NET, "Can't allocate pbuf for nbytes %d\n", nbytes);
                return -ENOMEM;
        }
 
@@ -2168,10 +2169,10 @@ again:
        }
 
        /* Received another push or reply that requires a larger buffer */
-       if (nnis < the_lnet.ln_push_target_nnis)
+       if (nbytes < the_lnet.ln_push_target_nbytes)
                goto again;
 
-       CDEBUG(D_NET, "nnis %d success\n", nnis);
+       CDEBUG(D_NET, "nbytes %d success\n", nbytes);
        return 0;
 }
 
@@ -2199,7 +2200,7 @@ int lnet_push_target_post(struct lnet_ping_buffer *pbuf,
 
        /* initialize md content */
        md.start     = &pbuf->pb_info;
-       md.length    = LNET_PING_INFO_SIZE(pbuf->pb_nnis);
+       md.length    = pbuf->pb_nbytes;
        md.threshold = 1;
        md.max_size  = 0;
        md.options   = LNET_MD_OP_PUT | LNET_MD_TRUNCATE;
@@ -2256,10 +2257,9 @@ static int lnet_push_target_init(void)
        LASSERT(rc == 0);
 
        /* Start at the required minimum, we'll enlarge if required. */
-       the_lnet.ln_push_target_nnis = LNET_INTERFACES_MIN;
+       the_lnet.ln_push_target_nbytes = LNET_PING_INFO_MIN_SIZE;
 
        rc = lnet_push_target_resize();
-
        if (rc) {
                LNetClearLazyPortal(LNET_RESERVED_PORTAL);
                the_lnet.ln_push_target_handler = NULL;
@@ -2286,7 +2286,7 @@ static void lnet_push_target_fini(void)
        /* Drop ref set by lnet_ping_buffer_alloc() */
        lnet_ping_buffer_decref(the_lnet.ln_push_target);
        the_lnet.ln_push_target = NULL;
-       the_lnet.ln_push_target_nnis = 0;
+       the_lnet.ln_push_target_nbytes = 0;
 
        LNetClearLazyPortal(LNET_RESERVED_PORTAL);
        lnet_assert_handler_unused(the_lnet.ln_push_target_handler);
@@ -2949,13 +2949,13 @@ void lnet_lib_exit(void)
 int
 LNetNIInit(lnet_pid_t requested_pid)
 {
-       int                     im_a_router = 0;
-       int                     rc;
-       int                     ni_count;
+       int im_a_router = 0;
+       int rc;
+       int ni_bytes;
        struct lnet_ping_buffer *pbuf;
-       struct lnet_handle_md   ping_mdh;
+       struct lnet_handle_md ping_mdh;
        LIST_HEAD(net_head);
-       struct lnet_net         *net;
+       struct lnet_net *net;
 
        mutex_lock(&the_lnet.ln_api_mutex);
 
@@ -3006,11 +3006,9 @@ LNetNIInit(lnet_pid_t requested_pid)
                        goto err_empty_list;
        }
 
-       ni_count = lnet_startup_lndnets(&net_head);
-       if (ni_count < 0) {
-               rc = ni_count;
+       rc = lnet_startup_lndnets(&net_head);
+       if (rc < 0)
                goto err_empty_list;
-       }
 
        if (!the_lnet.ln_nis_from_mod_params) {
                rc = lnet_parse_routes(lnet_get_routes(), &im_a_router);
@@ -3029,7 +3027,11 @@ LNetNIInit(lnet_pid_t requested_pid)
        the_lnet.ln_refcount = 1;
        /* Now I may use my own API functions... */
 
-       rc = lnet_ping_target_setup(&pbuf, &ping_mdh, ni_count, true);
+       ni_bytes = LNET_PING_INFO_HDR_SIZE;
+       list_for_each_entry(net, &the_lnet.ln_nets, net_list)
+               ni_bytes += lnet_get_net_ni_bytes_locked(net);
+
+       rc = lnet_ping_target_setup(&pbuf, &ping_mdh, ni_bytes, true);
        if (rc != 0)
                goto err_acceptor_stop;
 
@@ -3452,7 +3454,7 @@ static int lnet_add_net_common(struct lnet_net *net,
        struct lnet_ping_buffer *pbuf;
        struct lnet_remotenet *rnet;
        struct lnet_ni *ni;
-       int net_ni_count;
+       int net_ni_bytes;
        __u32 net_id;
        int rc;
 
@@ -3477,12 +3479,13 @@ static int lnet_add_net_common(struct lnet_net *net,
         * which will be added.
         *
         * since ni hasn't been configured yet, use
-        * lnet_get_net_ni_count_pre() which checks the net_ni_added list
+        * lnet_get_net_ni_bytes_pre() which checks the net_ni_added list
         */
-       net_ni_count = lnet_get_net_ni_count_pre(net);
+       net_ni_bytes = lnet_get_net_ni_bytes_pre(net);
 
        rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
-                                   net_ni_count + lnet_get_ni_count(),
+                                   LNET_PING_INFO_HDR_SIZE +
+                                   net_ni_bytes + lnet_get_ni_bytes(),
                                    false);
        if (rc < 0) {
                lnet_net_free(net);
@@ -3674,14 +3677,14 @@ int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf)
 
 int lnet_dyn_del_ni(struct lnet_ioctl_config_ni *conf)
 {
-       struct lnet_net  *net;
+       struct lnet_net *net;
        struct lnet_ni *ni;
        __u32 net_id = LNET_NIDNET(conf->lic_nid);
        struct lnet_ping_buffer *pbuf;
-       struct lnet_handle_md  ping_mdh;
-       int               rc;
-       int               net_count;
-       __u32             addr;
+       struct lnet_handle_md ping_mdh;
+       int net_bytes, rc;
+       bool net_empty;
+       u32 addr;
 
        /* don't allow userspace to shutdown the LOLND */
        if (LNET_NETTYP(net_id) == LOLND)
@@ -3706,14 +3709,15 @@ int lnet_dyn_del_ni(struct lnet_ioctl_config_ni *conf)
        addr = LNET_NIDADDR(conf->lic_nid);
        if (addr == 0) {
                /* remove the entire net */
-               net_count = lnet_get_net_ni_count_locked(net);
+               net_bytes = lnet_get_net_ni_bytes_locked(net);
 
                lnet_net_unlock(0);
 
                /* create and link a new ping info, before removing the old one */
                rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
-                                       lnet_get_ni_count() - net_count,
-                                       false);
+                                           LNET_PING_INFO_HDR_SIZE +
+                                           lnet_get_ni_bytes() - net_bytes,
+                                           false);
                if (rc != 0)
                        goto unlock_api_mutex;
 
@@ -3734,13 +3738,17 @@ int lnet_dyn_del_ni(struct lnet_ioctl_config_ni *conf)
                goto unlock_net;
        }
 
-       net_count = lnet_get_net_ni_count_locked(net);
+       net_bytes = lnet_get_net_ni_bytes_locked(net);
+       net_empty = list_is_singular(&net->net_ni_list);
 
        lnet_net_unlock(0);
 
        /* create and link a new ping info, before removing the old one */
        rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
-                                 lnet_get_ni_count() - 1, false);
+                                   (LNET_PING_INFO_HDR_SIZE +
+                                    lnet_get_ni_bytes() -
+                                    sizeof(pbuf->pb_info.pi_ni[0])),
+                                   false);
        if (rc != 0)
                goto unlock_api_mutex;
 
@@ -3751,7 +3759,7 @@ int lnet_dyn_del_ni(struct lnet_ioctl_config_ni *conf)
        lnet_ping_target_update(pbuf, ping_mdh);
 
        /* check if the net is empty and remove it if it is */
-       if (net_count == 1)
+       if (net_empty)
                lnet_shutdown_lndnet(net);
 
        goto unlock_api_mutex;
@@ -3829,13 +3837,12 @@ out_unlock_clean:
 }
 
 int
-lnet_dyn_del_net(__u32 net_id)
+lnet_dyn_del_net(u32 net_id)
 {
-       struct lnet_net  *net;
+       struct lnet_net *net;
        struct lnet_ping_buffer *pbuf;
        struct lnet_handle_md ping_mdh;
-       int               rc;
-       int               net_ni_count;
+       int net_ni_bytes, rc;
 
        /* don't allow userspace to shutdown the LOLND */
        if (LNET_NETTYP(net_id) == LOLND)
@@ -3856,13 +3863,15 @@ lnet_dyn_del_net(__u32 net_id)
                goto out;
        }
 
-       net_ni_count = lnet_get_net_ni_count_locked(net);
+       net_ni_bytes = lnet_get_net_ni_bytes_locked(net);
 
        lnet_net_unlock(0);
 
        /* create and link a new ping info, before removing the old one */
        rc = lnet_ping_target_setup(&pbuf, &ping_mdh,
-                                   lnet_get_ni_count() - net_ni_count, false);
+                                   LNET_PING_INFO_HDR_SIZE +
+                                   lnet_get_ni_bytes() - net_ni_bytes,
+                                   false);
        if (rc != 0)
                goto out;
 
@@ -4733,6 +4742,12 @@ lnet_ping_event_handler(struct lnet_event *event)
                complete(&pd->completion);
 }
 
+/* lnet_ping() only works with nid4 nids, so we can calculate
+ * size from number of nids
+ */
+#define LNET_PING_INFO_SIZE(NNIDS) \
+       offsetof(struct lnet_ping_info, pi_ni[NNIDS])
+
 static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
                     signed long timeout, struct lnet_process_id __user *ids,
                     int n_ids)
@@ -4742,6 +4757,7 @@ static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
        struct lnet_ping_buffer *pbuf;
        struct lnet_process_id tmpid;
        struct lnet_processid id;
+       int id_bytes;
        int i;
        int nob;
        int rc;
@@ -4761,13 +4777,14 @@ static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
        if (id4.pid == LNET_PID_ANY)
                id4.pid = LNET_PID_LUSTRE;
 
-       pbuf = lnet_ping_buffer_alloc(n_ids, GFP_NOFS);
+       id_bytes = LNET_PING_INFO_SIZE(n_ids);
+       pbuf = lnet_ping_buffer_alloc(id_bytes, GFP_NOFS);
        if (!pbuf)
                return -ENOMEM;
 
        /* initialize md content */
        md.start     = &pbuf->pb_info;
-       md.length    = LNET_PING_INFO_SIZE(n_ids);
+       md.length    = id_bytes;
        md.threshold = 2; /* GET/REPLY */
        md.max_size  = 0;
        md.options   = LNET_MD_TRUNCATE;
@@ -4805,7 +4822,7 @@ static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
        }
 
        nob = pd.rc;
-       LASSERT(nob >= 0 && nob <= LNET_PING_INFO_SIZE(n_ids));
+       LASSERT(nob >= 0 && nob <= id_bytes);
 
        rc = -EPROTO;           /* if I can't parse... */
 
@@ -4829,20 +4846,21 @@ static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
                goto fail_ping_buffer_decref;
        }
 
-       if (nob < LNET_PING_INFO_SIZE(0)) {
-               CERROR("%s: Short reply %d(%d min)\n",
-                      libcfs_idstr(&id),
-                      nob, (int)LNET_PING_INFO_SIZE(0));
+       /* Test if smaller than lnet_pinginfo with no pi_ni status info */
+       if (nob < LNET_PING_INFO_HDR_SIZE) {
+               CERROR("%s: Short reply %d(%lu min)\n",
+                      libcfs_idstr(&id), nob, LNET_PING_INFO_HDR_SIZE);
                goto fail_ping_buffer_decref;
        }
 
-       if (pbuf->pb_info.pi_nnis < n_ids)
+       if (pbuf->pb_info.pi_nnis < n_ids) {
                n_ids = pbuf->pb_info.pi_nnis;
+               id_bytes = lnet_ping_info_size(&pbuf->pb_info);
+       }
 
-       if (nob < LNET_PING_INFO_SIZE(n_ids)) {
+       if (nob < id_bytes) {
                CERROR("%s: Short reply %d(%d expected)\n",
-                      libcfs_idstr(&id),
-                      nob, (int)LNET_PING_INFO_SIZE(n_ids));
+                      libcfs_idstr(&id), nob, id_bytes);
                goto fail_ping_buffer_decref;
        }
 
@@ -4861,6 +4879,7 @@ static int lnet_ping(struct lnet_process_id id4, struct lnet_nid *src_nid,
        lnet_ping_buffer_decref(pbuf);
        return rc;
 }
+#undef LNET_PING_INFO_SIZE
 
 static int
 lnet_discover(struct lnet_process_id id4, __u32 force,

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index da31094..4bc7199 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -3584,7 +3584,7 @@ lnet_recover_local_nis(void)
 
                        ev_info->mt_type = MT_TYPE_LOCAL_NI;
                        ev_info->mt_nid = nid;
-                       rc = lnet_send_ping(&nid, &mdh, LNET_INTERFACES_MIN,
+                       rc = lnet_send_ping(&nid, &mdh, LNET_PING_INFO_MIN_SIZE,
                                            ev_info, the_lnet.ln_mt_handler,
                                            true);
                        /* lookup the nid again */
@@ -3817,7 +3817,7 @@ lnet_recover_peer_nis(void)
 
                        ev_info->mt_type = MT_TYPE_PEER_NI;
                        ev_info->mt_nid = nid;
-                       rc = lnet_send_ping(&nid, &mdh, LNET_INTERFACES_MIN,
+                       rc = lnet_send_ping(&nid, &mdh, LNET_PING_INFO_MIN_SIZE,
                                            ev_info, the_lnet.ln_mt_handler,
                                            true);
                        lnet_net_lock(0);
@@ -3929,7 +3929,7 @@ lnet_monitor_thread(void *arg)
  */
 int
 lnet_send_ping(struct lnet_nid *dest_nid,
-              struct lnet_handle_md *mdh, int nnis,
+              struct lnet_handle_md *mdh, int bytes,
               void *user_data, lnet_handler_t handler, bool recovery)
 {
        struct lnet_md md = { NULL };
@@ -3942,7 +3942,7 @@ lnet_send_ping(struct lnet_nid *dest_nid,
                goto fail_error;
        }
 
-       pbuf = lnet_ping_buffer_alloc(nnis, GFP_NOFS);
+       pbuf = lnet_ping_buffer_alloc(bytes, GFP_NOFS);
        if (!pbuf) {
                rc = ENOMEM;
                goto fail_error;
@@ -3950,7 +3950,7 @@ lnet_send_ping(struct lnet_nid *dest_nid,
 
        /* initialize md content */
        md.start     = &pbuf->pb_info;
-       md.length    = LNET_PING_INFO_SIZE(nnis);
+       md.length    = bytes;
        md.threshold = 2; /* GET/REPLY */
        md.max_size  = 0;
        md.options   = LNET_MD_TRUNCATE | LNET_MD_TRACK_RESPONSE;

diff --git a/lnet/lnet/lib-msg.c b/lnet/lnet/lib-msg.c
index 36a29cc..f54acbf 100644 (file)
--- a/lnet/lnet/lib-msg.c
+++ b/lnet/lnet/lib-msg.c
@@ -817,6 +817,8 @@ lnet_health_check(struct lnet_msg *msg)
         * messages with a health status != OK.
         */
        if (hstatus != LNET_MSG_STATUS_OK) {
+               struct lnet_ping_info *pi;
+
                /* Don't further decrement the health value if a recovery
                 * message failed.
                 */
@@ -826,11 +828,10 @@ lnet_health_check(struct lnet_msg *msg)
                        handle_local_health = handle_remote_health = true;
 
                /* For local failures, health/recovery/resends are not needed if
-                * I only have a single (non-lolnd) interface. NB: pb_nnis
-                * includes the lolnd interface, so a single-rail node would
-                * have pb_nnis == 2.
+                * I only have a single (non-lolnd) interface.
                 */
-               if (the_lnet.ln_ping_target->pb_nnis <= 2) {
+               pi = &the_lnet.ln_ping_target->pb_info;
+               if (pi->pi_nnis <= 2) {
                        handle_local_health = false;
                        attempt_local_resend = false;
                }
@@ -840,9 +841,8 @@ lnet_health_check(struct lnet_msg *msg)
                /* For remote failures, health/recovery/resends are not needed
                 * if the peer only has a single interface. Special case for
                 * routers where we rely on health feature to manage route
-                * aliveness. NB: unlike pb_nnis above, lp_nnis does _not_
-                * include the lolnd, so a single-rail node would have
-                * lp_nnis == 1.
+                * aliveness. NB: lp_nnis does _not_ include the lolnd, so a
+                * single-rail node would have lp_nnis == 1.
                 */
                if (lpni && lpni->lpni_peer_net &&
                    lpni->lpni_peer_net->lpn_peer &&

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 1fcb697..1650b9f 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -2264,6 +2264,7 @@ void lnet_peer_push_event(struct lnet_event *ev)
 {
        struct lnet_ping_buffer *pbuf;
        struct lnet_peer *lp;
+       int infobytes;
 
        pbuf = LNET_PING_INFO_TO_BUFFER(ev->md_start + ev->offset);
 
@@ -2307,12 +2308,12 @@ void lnet_peer_push_event(struct lnet_event *ev)
                goto out;
        }
 
-       /*
-        * Make sure we'll allocate the correct size ping buffer when
+       /* Make sure we'll allocate the correct size ping buffer when
         * pinging the peer.
         */
-       if (lp->lp_data_nnis < pbuf->pb_info.pi_nnis)
-               lp->lp_data_nnis = pbuf->pb_info.pi_nnis;
+       infobytes = lnet_ping_info_size(&pbuf->pb_info);
+       if (lp->lp_data_bytes < infobytes)
+               lp->lp_data_bytes = infobytes;
 
        /*
         * A non-Multi-Rail peer is not supposed to be capable of
@@ -2375,19 +2376,17 @@ void lnet_peer_push_event(struct lnet_event *ev)
                lnet_peer_clr_non_mr_pref_nids(lp);
        }
 
-       /*
-        * Check for truncation of the Put message. Clear the
+       /* Check for truncation of the Put message. Clear the
         * NIDS_UPTODATE flag and set FORCE_PING to trigger a ping,
         * and tell discovery to allocate a bigger buffer.
         */
        if (ev->mlength < ev->rlength) {
-               if (the_lnet.ln_push_target_nnis < pbuf->pb_info.pi_nnis)
-                       the_lnet.ln_push_target_nnis = pbuf->pb_info.pi_nnis;
+               if (the_lnet.ln_push_target_nbytes < infobytes)
+                       the_lnet.ln_push_target_nbytes = infobytes;
                lp->lp_state &= ~LNET_PEER_NIDS_UPTODATE;
                lp->lp_state |= LNET_PEER_FORCE_PING;
-               CDEBUG(D_NET, "Truncated Push from %s (%d nids)\n",
-                      libcfs_nidstr(&lp->lp_primary_nid),
-                      pbuf->pb_info.pi_nnis);
+               CDEBUG(D_NET, "Truncated Push from %s (%d bytes)\n",
+                      libcfs_nidstr(&lp->lp_primary_nid), infobytes);
                goto out;
        }
 
@@ -2395,8 +2394,7 @@ void lnet_peer_push_event(struct lnet_event *ev)
        lp->lp_peer_seqno = LNET_PING_BUFFER_SEQNO(pbuf);
        lp->lp_state &= ~LNET_PEER_NIDS_UPTODATE;
 
-       /*
-        * If there is data present that hasn't been processed yet,
+       /* If there is data present that hasn't been processed yet,
         * we'll replace it if the Put contained newer data and it
         * fits. We're racing with a Ping or earlier Push in this
         * case.
@@ -2404,9 +2402,9 @@ void lnet_peer_push_event(struct lnet_event *ev)
        if (lp->lp_state & LNET_PEER_DATA_PRESENT) {
                if (LNET_PING_BUFFER_SEQNO(pbuf) >
                        LNET_PING_BUFFER_SEQNO(lp->lp_data) &&
-                   pbuf->pb_info.pi_nnis <= lp->lp_data->pb_nnis) {
+                   infobytes <= lp->lp_data->pb_nbytes) {
                        memcpy(&lp->lp_data->pb_info, &pbuf->pb_info,
-                              LNET_PING_INFO_SIZE(pbuf->pb_info.pi_nnis));
+                              infobytes);
                        CDEBUG(D_NET, "Ping/Push race from %s: %u vs %u\n",
                              libcfs_nidstr(&lp->lp_primary_nid),
                              LNET_PING_BUFFER_SEQNO(pbuf),
@@ -2420,7 +2418,7 @@ void lnet_peer_push_event(struct lnet_event *ev)
         * the Push and set FORCE_PING to force the discovery
         * thread to fix the problem by pinging the peer.
         */
-       lp->lp_data = lnet_ping_buffer_alloc(lp->lp_data_nnis, GFP_ATOMIC);
+       lp->lp_data = lnet_ping_buffer_alloc(lp->lp_data_bytes, GFP_ATOMIC);
        if (!lp->lp_data) {
                lp->lp_state |= LNET_PEER_FORCE_PING;
                CDEBUG(D_NET, "Cannot allocate Push buffer for %s %u\n",
@@ -2430,8 +2428,7 @@ void lnet_peer_push_event(struct lnet_event *ev)
        }
 
        /* Success */
-       memcpy(&lp->lp_data->pb_info, &pbuf->pb_info,
-              LNET_PING_INFO_SIZE(pbuf->pb_info.pi_nnis));
+       memcpy(&lp->lp_data->pb_info, &pbuf->pb_info, infobytes);
        lp->lp_state |= LNET_PEER_DATA_PRESENT;
        CDEBUG(D_NET, "Received Push %s %u\n",
               libcfs_nidstr(&lp->lp_primary_nid),
@@ -2596,6 +2593,7 @@ static void
 lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev)
 {
        struct lnet_ping_buffer *pbuf;
+       int infobytes;
        int rc;
 
        spin_lock(&lp->lp_lock);
@@ -2708,25 +2706,24 @@ lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev)
                }
        }
 
+       infobytes = lnet_ping_info_size(&pbuf->pb_info);
        /*
         * Make sure we'll allocate the correct size ping buffer when
         * pinging the peer.
         */
-       if (lp->lp_data_nnis < pbuf->pb_info.pi_nnis)
-               lp->lp_data_nnis = pbuf->pb_info.pi_nnis;
+       if (lp->lp_data_bytes < infobytes)
+               lp->lp_data_bytes = infobytes;
 
-       /*
-        * Check for truncation of the Reply. Clear PING_SENT and set
+       /* Check for truncation of the Reply. Clear PING_SENT and set
         * PING_FAILED to trigger a retry.
         */
-       if (pbuf->pb_nnis < pbuf->pb_info.pi_nnis) {
-               if (the_lnet.ln_push_target_nnis < pbuf->pb_info.pi_nnis)
-                       the_lnet.ln_push_target_nnis = pbuf->pb_info.pi_nnis;
+       if (pbuf->pb_nbytes < infobytes) {
+               if (the_lnet.ln_push_target_nbytes < infobytes)
+                       the_lnet.ln_push_target_nbytes = infobytes;
                lp->lp_state |= LNET_PEER_PING_FAILED;
                lp->lp_ping_error = 0;
-               CDEBUG(D_NET, "Truncated Reply from %s (%d nids)\n",
-                      libcfs_nidstr(&lp->lp_primary_nid),
-                      pbuf->pb_info.pi_nnis);
+               CDEBUG(D_NET, "Truncated Reply from %s (%d bytes)\n",
+                      libcfs_nidstr(&lp->lp_primary_nid), infobytes);
                goto out;
        }
 
@@ -3408,7 +3405,7 @@ __must_hold(&lp->lp_lock)
 static int lnet_peer_send_ping(struct lnet_peer *lp)
 __must_hold(&lp->lp_lock)
 {
-       int nnis;
+       int bytes;
        int rc;
        int cpt;
 
@@ -3421,13 +3418,11 @@ __must_hold(&lp->lp_lock)
        lnet_peer_addref_locked(lp);
        lnet_net_unlock(cpt);
 
-       nnis = max(lp->lp_data_nnis, LNET_INTERFACES_MIN);
+       bytes = max_t(int, lp->lp_data_bytes, LNET_PING_INFO_MIN_SIZE);
 
-       rc = lnet_send_ping(&lp->lp_primary_nid, &lp->lp_ping_mdh, nnis, lp,
+       rc = lnet_send_ping(&lp->lp_primary_nid, &lp->lp_ping_mdh, bytes, lp,
                            the_lnet.ln_dc_handler, false);
-
-       /*
-        * if LNetMDBind in lnet_send_ping fails we need to decrement the
+       /* if LNetMDBind in lnet_send_ping fails we need to decrement the
         * refcount on the peer, otherwise LNetMDUnlink will be called
         * which will eventually do that.
         */
@@ -3535,7 +3530,7 @@ __must_hold(&lp->lp_lock)
 
        /* Push source MD */
        md.start     = &pbuf->pb_info;
-       md.length    = LNET_PING_INFO_SIZE(pbuf->pb_nnis);
+       md.length    = pbuf->pb_nbytes;
        md.threshold = 2; /* Put/Ack */
        md.max_size  = 0;
        md.options   = LNET_MD_TRACK_RESPONSE;