LU-9480 lnet: add lnet_interfaces_max tunable

Add an lnet_interfaces_max tunable value, that describes the maximum
number of interfaces per node. This tunable is primarily useful for
sanity checks prior to allocating memory.

Allow lnet_interfaces_max to be set and get from the sysfs interface.

Add LNET_INTERFACES_MIN, value 16, as the minimum value.

Add LNET_INTERFACES_MAX_DEFAULT, value 200, as the default value. This
value was chosen to ensure that the size of an LNet ping message with
any associated LND overhead would fit in 4096 bytes.

(The LNET_INTERFACES_MAX name was not reused to allow for the early
detection of issues when merging code that uses it.)

Rename LNET_NUM_INTERFACES to LNET_INTERFACES_NUM

Test-Parameters: trivial
Signed-off-by: Olaf Weber <olaf@sgi.com>
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I9bdc72cc688a414f7658fed93f84c9885c8342be
Reviewed-on: https://review.whamcloud.com/25770

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index d0095a6..1797641 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -404,7 +404,7 @@ typedef struct lnet_ni {
         * equivalent interfaces to use
         * This is an array because socklnd bonding can still be configured
         */
-       char                    *ni_interfaces[LNET_NUM_INTERFACES];
+       char                    *ni_interfaces[LNET_INTERFACES_NUM];
        struct net              *ni_net_ns;     /* original net namespace */
 } lnet_ni_t;
 

diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h
index 081f0ea..ea54f9f 100644 (file)
--- a/lnet/include/uapi/linux/lnet/lnet-dlc.h
+++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h
@@ -90,7 +90,7 @@ struct lnet_ioctl_config_lnd_tunables {
 };
 
 struct lnet_ioctl_net_config {
-       char ni_interfaces[LNET_NUM_INTERFACES][LNET_MAX_STR_LEN];
+       char ni_interfaces[LNET_INTERFACES_NUM][LNET_MAX_STR_LEN];
        __u32 ni_status;
        __u32 ni_cpts[LNET_MAX_SHOW_NUM_CPT];
        char cfg_bulk[0];
@@ -193,7 +193,7 @@ struct lnet_ioctl_element_msg_stats {
 struct lnet_ioctl_config_ni {
        struct libcfs_ioctl_hdr lic_cfg_hdr;
        lnet_nid_t              lic_nid;
-       char                    lic_ni_intf[LNET_NUM_INTERFACES][LNET_MAX_STR_LEN];
+       char                    lic_ni_intf[LNET_INTERFACES_NUM][LNET_MAX_STR_LEN];
        char                    lic_legacy_ip2nets[LNET_MAX_STR_LEN];
        __u32                   lic_cpts[LNET_MAX_SHOW_NUM_CPT];
        __u32                   lic_ncpts;

diff --git a/lnet/include/uapi/linux/lnet/lnet-types.h b/lnet/include/uapi/linux/lnet/lnet-types.h
index b10e3a8..520f8ff 100644 (file)
--- a/lnet/include/uapi/linux/lnet/lnet-types.h
+++ b/lnet/include/uapi/linux/lnet/lnet-types.h
@@ -247,7 +247,12 @@ typedef struct lnet_counters {
  * defined here because it is used in LNet data structures that are
  * common to all LNDs.
  */
-#define LNET_NUM_INTERFACES    16
+#define LNET_INTERFACES_NUM    16
+
+/* The minimum number of interfaces per node supported by LNet. */
+#define LNET_INTERFACES_MIN    16
+/* The default - arbitrary - value of the lnet_max_interfaces tunable. */
+#define LNET_INTERFACES_MAX_DEFAULT    200
 
 /**
  * Objects maintained by the LNet are accessed through handles. Handle types

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 110b6e6..493e73c 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -3279,7 +3279,7 @@ kiblnd_startup(struct lnet_ni *ni)
        if (ni->ni_interfaces[0] != NULL) {
                /* Use the IPoIB interface specified in 'networks=' */
 
-               CLASSERT(LNET_NUM_INTERFACES > 1);
+               CLASSERT(LNET_INTERFACES_NUM > 1);
                if (ni->ni_interfaces[1] != NULL) {
                        CERROR("Multiple interfaces not supported\n");
                        goto failed;

diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c
index 8e5b675..56045fb 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.c
+++ b/lnet/klnds/socklnd/socklnd.c
@@ -51,7 +51,7 @@ ksocknal_ip2iface(struct lnet_ni *ni, __u32 ip)
        struct ksock_interface *iface;
 
        for (i = 0; i < net->ksnn_ninterfaces; i++) {
-               LASSERT(i < LNET_NUM_INTERFACES);
+               LASSERT(i < LNET_INTERFACES_NUM);
                iface = &net->ksnn_interfaces[i];
 
                if (iface->ksni_ipaddr == ip)
@@ -220,7 +220,7 @@ ksocknal_unlink_peer_locked(struct ksock_peer_ni *peer_ni)
        struct ksock_interface *iface;
 
        for (i = 0; i < peer_ni->ksnp_n_passive_ips; i++) {
-               LASSERT(i < LNET_NUM_INTERFACES);
+               LASSERT(i < LNET_INTERFACES_NUM);
                ip = peer_ni->ksnp_passive_ips[i];
 
                iface = ksocknal_ip2iface(peer_ni->ksnp_ni, ip);
@@ -705,7 +705,7 @@ ksocknal_local_ipvec(struct lnet_ni *ni, __u32 *ipaddrs)
        read_lock(&ksocknal_data.ksnd_global_lock);
 
        nip = net->ksnn_ninterfaces;
-       LASSERT(nip <= LNET_NUM_INTERFACES);
+       LASSERT(nip <= LNET_INTERFACES_NUM);
 
        /*
         * Only offer interfaces for additional connections if I have
@@ -784,8 +784,8 @@ ksocknal_select_ips(struct ksock_peer_ni *peer_ni, __u32 *peerips, int n_peerips
 
        write_lock_bh(global_lock);
 
-       LASSERT(n_peerips <= LNET_NUM_INTERFACES);
-       LASSERT(net->ksnn_ninterfaces <= LNET_NUM_INTERFACES);
+       LASSERT(n_peerips <= LNET_INTERFACES_NUM);
+       LASSERT(net->ksnn_ninterfaces <= LNET_INTERFACES_NUM);
 
        /* Only match interfaces for additional connections
          * if I have > 1 interface */
@@ -895,7 +895,7 @@ ksocknal_create_routes(struct ksock_peer_ni *peer_ni, int port,
                 return;
         }
 
-       LASSERT(npeer_ipaddrs <= LNET_NUM_INTERFACES);
+       LASSERT(npeer_ipaddrs <= LNET_INTERFACES_NUM);
 
         for (i = 0; i < npeer_ipaddrs; i++) {
                 if (newroute != NULL) {
@@ -932,7 +932,7 @@ ksocknal_create_routes(struct ksock_peer_ni *peer_ni, int port,
                best_nroutes = 0;
                best_netmatch = 0;
 
-               LASSERT(net->ksnn_ninterfaces <= LNET_NUM_INTERFACES);
+               LASSERT(net->ksnn_ninterfaces <= LNET_INTERFACES_NUM);
 
                /* Select interface to connect from */
                for (j = 0; j < net->ksnn_ninterfaces; j++) {
@@ -1077,7 +1077,7 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route,
        atomic_set (&conn->ksnc_tx_nob, 0);
 
        LIBCFS_ALLOC(hello, offsetof(struct ksock_hello_msg,
-                                    kshm_ips[LNET_NUM_INTERFACES]));
+                                    kshm_ips[LNET_INTERFACES_NUM]));
         if (hello == NULL) {
                 rc = -ENOMEM;
                 goto failed_1;
@@ -1336,7 +1336,7 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route,
         }
 
        LIBCFS_FREE(hello, offsetof(struct ksock_hello_msg,
-                                   kshm_ips[LNET_NUM_INTERFACES]));
+                                   kshm_ips[LNET_INTERFACES_NUM]));
 
         /* setup the socket AFTER I've received hello (it disables
          * SO_LINGER).  I might call back to the acceptor who may want
@@ -1420,7 +1420,7 @@ failed_2:
 failed_1:
        if (hello != NULL)
                LIBCFS_FREE(hello, offsetof(struct ksock_hello_msg,
-                                           kshm_ips[LNET_NUM_INTERFACES]));
+                                           kshm_ips[LNET_INTERFACES_NUM]));
 
        LIBCFS_FREE(conn, sizeof(*conn));
 
@@ -1984,7 +1984,7 @@ ksocknal_add_interface(struct lnet_ni *ni, __u32 ipaddress, __u32 netmask)
        if (iface != NULL) {
                /* silently ignore dups */
                rc = 0;
-       } else if (net->ksnn_ninterfaces == LNET_NUM_INTERFACES) {
+       } else if (net->ksnn_ninterfaces == LNET_INTERFACES_NUM) {
                rc = -ENOSPC;
        } else {
                iface = &net->ksnn_interfaces[net->ksnn_ninterfaces++];
@@ -2653,7 +2653,7 @@ ksocknal_enumerate_interfaces(struct ksock_net *net)
                         continue;
                 }
 
-               if (j == LNET_NUM_INTERFACES) {
+               if (j == LNET_INTERFACES_NUM) {
                        CWARN("Ignoring interface %s (too many interfaces)\n",
                              names[i]);
                        continue;
@@ -2834,7 +2834,7 @@ ksocknal_startup(struct lnet_ni *ni)
 
                net->ksnn_ninterfaces = 1;
        } else {
-               for (i = 0; i < LNET_NUM_INTERFACES; i++) {
+               for (i = 0; i < LNET_INTERFACES_NUM; i++) {
                        int up;
 
                        if (ni->ni_interfaces[i] == NULL)

diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h
index ff71b3a..c4506c9 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.h
+++ b/lnet/klnds/socklnd/socklnd.h
@@ -178,7 +178,7 @@ struct ksock_net {
        int               ksnn_npeers;          /* # peers */
        int               ksnn_shutdown;        /* shutting down? */
        int               ksnn_ninterfaces;     /* IP interfaces */
-       struct ksock_interface ksnn_interfaces[LNET_NUM_INTERFACES];
+       struct ksock_interface ksnn_interfaces[LNET_INTERFACES_NUM];
 };
 
 /** connd timeout */
@@ -418,7 +418,7 @@ struct ksock_peer_ni {
        time64_t                ksnp_send_keepalive; /* time to send keepalive */
        struct lnet_ni       *ksnp_ni;       /* which network */
        int                   ksnp_n_passive_ips; /* # of... */
-       __u32                 ksnp_passive_ips[LNET_NUM_INTERFACES]; /* preferred local interfaces */
+       __u32                 ksnp_passive_ips[LNET_INTERFACES_NUM]; /* preferred local interfaces */
 };
 
 struct ksock_connreq {

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index af6976f..f8b0f06 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -1668,7 +1668,7 @@ ksocknal_send_hello(struct lnet_ni *ni, struct ksock_conn *conn,
        /* CAVEAT EMPTOR: this byte flips 'ipaddrs' */
        struct ksock_net *net = (struct ksock_net *)ni->ni_data;
 
-       LASSERT(hello->kshm_nips <= LNET_NUM_INTERFACES);
+       LASSERT(hello->kshm_nips <= LNET_INTERFACES_NUM);
 
        /* rely on caller to hold a ref on socket so it wouldn't disappear */
        LASSERT(conn->ksnc_proto != NULL);

diff --git a/lnet/klnds/socklnd/socklnd_proto.c b/lnet/klnds/socklnd/socklnd_proto.c
index 3064dc3..ca106a3 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_proto.c
+++ b/lnet/klnds/socklnd/socklnd_proto.c
@@ -608,7 +608,7 @@ ksocknal_recv_hello_v1(struct ksock_conn *conn, struct ksock_hello_msg *hello,
         hello->kshm_nips            = le32_to_cpu (hdr->payload_length) /
                                          sizeof (__u32);
 
-       if (hello->kshm_nips > LNET_NUM_INTERFACES) {
+       if (hello->kshm_nips > LNET_INTERFACES_NUM) {
                CERROR("Bad nips %d from ip %pI4h\n",
                       hello->kshm_nips, &conn->ksnc_ipaddr);
                rc = -EPROTO;
@@ -678,7 +678,7 @@ ksocknal_recv_hello_v2(struct ksock_conn *conn, struct ksock_hello_msg *hello,
                 __swab32s(&hello->kshm_nips);
         }
 
-       if (hello->kshm_nips > LNET_NUM_INTERFACES) {
+       if (hello->kshm_nips > LNET_INTERFACES_NUM) {
                CERROR("Bad nips %d from ip %pI4h\n",
                       hello->kshm_nips, &conn->ksnc_ipaddr);
                return -EPROTO;

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 1a5aa34..2db2d7a 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -33,6 +33,7 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #include <linux/log2.h>
 #include <linux/ktime.h>
+#include <linux/moduleparam.h>
 
 #include <lnet/lib-lnet.h>
 
@@ -67,6 +68,13 @@ module_param(lnet_numa_range, uint, 0444);
 MODULE_PARM_DESC(lnet_numa_range,
                "NUMA range to consider during Multi-Rail selection");
 
+static int lnet_interfaces_max = LNET_INTERFACES_MAX_DEFAULT;
+static int intf_max_set(const char *val, struct kernel_param *kp);
+module_param_call(lnet_interfaces_max, intf_max_set, param_get_int,
+                 &lnet_interfaces_max, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(lnet_interfaces_max,
+               "Maximum number of interfaces in a node.");
+
 /*
  * This sequence number keeps track of how many times DLC was used to
  * update the local NIs. It is incremented when a NI is added or
@@ -79,6 +87,28 @@ static atomic_t lnet_dlc_seq_no = ATOMIC_INIT(0);
 static int lnet_ping(struct lnet_process_id id, signed long timeout,
                     struct lnet_process_id __user *ids, int n_ids);
 
+static int
+intf_max_set(const char *val, struct kernel_param *kp)
+{
+       int value, rc;
+
+       rc = kstrtoint(val, 0, &value);
+       if (rc) {
+               CERROR("Invalid module parameter value for 'lnet_interfaces_max'\n");
+               return rc;
+       }
+
+       if (value < LNET_INTERFACES_MIN) {
+               CWARN("max interfaces provided are too small, setting to %d\n",
+                     LNET_INTERFACES_MIN);
+               value = LNET_INTERFACES_MIN;
+       }
+
+       *(int *)kp->arg = value;
+
+       return 0;
+}
+
 static char *
 lnet_get_routes(void)
 {
@@ -3010,23 +3040,23 @@ static int lnet_ping(struct lnet_process_id id, signed long timeout,
        struct lnet_handle_md mdh;
        struct lnet_event event;
        struct lnet_md md = { NULL };
-       int                  which;
-       int                  unlinked = 0;
-       int                  replied = 0;
+       int which;
+       int unlinked = 0;
+       int replied = 0;
        const signed long a_long_time = msecs_to_jiffies(60 * MSEC_PER_SEC);
-       int                  infosz;
+       int infosz;
        struct lnet_ping_info *info;
        struct lnet_process_id tmpid;
-       int                  i;
-       int                  nob;
-       int                  rc;
-       int                  rc2;
-       sigset_t         blocked;
+       int i;
+       int nob;
+       int rc;
+       int rc2;
+       sigset_t blocked;
 
        infosz = offsetof(struct lnet_ping_info, pi_ni[n_ids]);
 
        /* n_ids limit is arbitrary */
-       if (n_ids <= 0 || n_ids > 20 || id.nid == LNET_NID_ANY)
+       if (n_ids <= 0 || n_ids > lnet_interfaces_max || id.nid == LNET_NID_ANY)
                return -EINVAL;
 
        if (id.pid == LNET_PID_ANY)

diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c
index fbbc18c..4cdd845 100644 (file)
--- a/lnet/lnet/config.c
+++ b/lnet/lnet/config.c
@@ -121,10 +121,10 @@ lnet_ni_unique_net(struct list_head *nilist, char *iface)
 /* check that the NI is unique to the interfaces with in the same NI.
  * This is only a consideration if use_tcp_bonding is set */
 static bool
-lnet_ni_unique_ni(char *iface_list[LNET_NUM_INTERFACES], char *iface)
+lnet_ni_unique_ni(char *iface_list[LNET_INTERFACES_NUM], char *iface)
 {
        int i;
-       for (i = 0; i < LNET_NUM_INTERFACES; i++) {
+       for (i = 0; i < LNET_INTERFACES_NUM; i++) {
                if (iface_list[i] != NULL &&
                    strncmp(iface_list[i], iface, strlen(iface)) == 0)
                        return false;
@@ -307,7 +307,7 @@ lnet_ni_free(struct lnet_ni *ni)
        if (ni->ni_cpts != NULL)
                cfs_expr_list_values_free(ni->ni_cpts, ni->ni_ncpts);
 
-       for (i = 0; i < LNET_NUM_INTERFACES &&
+       for (i = 0; i < LNET_INTERFACES_NUM &&
                    ni->ni_interfaces[i] != NULL; i++) {
                LIBCFS_FREE(ni->ni_interfaces[i],
                            strlen(ni->ni_interfaces[i]) + 1);
@@ -407,11 +407,11 @@ lnet_ni_add_interface(struct lnet_ni *ni, char *iface)
         * can free the tokens at the end of the function.
         * The newly allocated ni_interfaces[] can be
         * freed when freeing the NI */
-       while (niface < LNET_NUM_INTERFACES &&
+       while (niface < LNET_INTERFACES_NUM &&
               ni->ni_interfaces[niface] != NULL)
                niface++;
 
-       if (niface >= LNET_NUM_INTERFACES) {
+       if (niface >= LNET_INTERFACES_NUM) {
                LCONSOLE_ERROR_MSG(0x115, "Too many interfaces "
                                   "for net %s\n",
                                   libcfs_net2str(LNET_NIDNET(ni->ni_nid)));

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index fc06e96..885ef06 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -1623,7 +1623,7 @@ int lustre_lnet_show_net(char *nw, int detail, int seq_no,
                        if (interfaces == NULL)
                                goto out;
 
-                       for (j = 0; j < LNET_NUM_INTERFACES; j++) {
+                       for (j = 0; j < LNET_INTERFACES_NUM; j++) {
                                if (strlen(ni_data->lic_ni_intf[j]) > 0) {
                                        snprintf(str_buf,
                                                 sizeof(str_buf), "%d", j);