LU-12815 socklnd: set conns_per_peer based on link speed

Specifying conns_per_peer=0 for a ni is now used to set
the conns_per_peer as a function of the corresponding link speed
as follows:
	conns_per_peer = (ilog2(Gbps) / 2 + 1)

Listed below are the resulting defaults for common link speeds:
	100Gbps, 200Gbps -> 4
        50Gbps		 -> 3
        5Gbps, 10Gbps	 -> 2
        less than 4Gbps  -> 1

Test-Parameters: trivial testlist=sanity-lnet
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: Ief2b33a796c180d8669bd5796b3e35ec748423a5
Reviewed-on: https://review.whamcloud.com/44417
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index cd0bc82..c6be763 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -570,6 +570,17 @@ AS_IF([test $ENABLEO2IB != "no"], [
                        [rdma_connect_locked is defined])
        ])
 
+       # ethtool_link_settings was added in Linux 4.6
+       LB_CHECK_COMPILE([if 'ethtool_link_settings' exists],
+       ethtool_link_settings, [
+               #include <linux/ethtool.h>
+       ],[
+               struct ethtool_link_ksettings cmd;
+       ],[
+               AC_DEFINE(HAVE_ETHTOOL_LINK_SETTINGS, 1,
+                       [ethtool_link_settings is defined])
+       ])
+
        EXTRA_CHECK_INCLUDE=""
        AC_DEFUN([LN_CONFIG_O2IB_SRC], [])
        AC_DEFUN([LN_CONFIG_O2IB_RESULTS], [])

diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c
index ab4adb4..ec7e069 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.c
+++ b/lnet/klnds/socklnd/socklnd.c
@@ -2367,7 +2367,6 @@ fail_0:
        return -ENETDOWN;
 }
 
-
 static void __exit ksocklnd_exit(void)
 {
        lnet_unregister_lnd(&the_ksocklnd);

diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c
index 72ae1ac..967575a 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_modparams.c
+++ b/lnet/klnds/socklnd/socklnd_modparams.c
@@ -25,6 +25,10 @@
 #if defined(__x86_64__) || defined(__i386__)
 #include <asm/hypervisor.h>
 #endif
+#ifdef HAVE_ETHTOOL_LINK_SETTINGS
+#include <linux/inetdevice.h>
+#include <linux/ethtool.h>
+#endif
 
 #define CURRENT_LND_VERSION 1
 
@@ -178,6 +182,81 @@ static inline bool is_native_host(void)
 struct ksock_tunables ksocknal_tunables;
 static struct lnet_ioctl_config_socklnd_tunables default_tunables;
 
+#ifdef HAVE_ETHTOOL_LINK_SETTINGS
+static int ksocklnd_ni_get_eth_intf_speed(struct lnet_ni *ni)
+{
+       struct net_device *dev;
+       int intf_idx = -1;
+       int ret = -1;
+
+       DECLARE_CONST_IN_IFADDR(ifa);
+
+       rtnl_lock();
+       for_each_netdev(ni->ni_net_ns, dev) {
+               int flags = dev_get_flags(dev);
+               struct in_device *in_dev;
+
+               if (flags & IFF_LOOPBACK) /* skip the loopback IF */
+                       continue;
+
+               if (!(flags & IFF_UP))
+                       continue;
+
+               in_dev = __in_dev_get_rcu(dev);
+               if (!in_dev)
+                       continue;
+
+               in_dev_for_each_ifa_rcu(ifa, in_dev) {
+                       if (strcmp(ifa->ifa_label, ni->ni_interface) == 0)
+                               intf_idx = dev->ifindex;
+               }
+               endfor_ifa(in_dev);
+
+               if (intf_idx >= 0)
+                       break;
+       }
+       if (intf_idx >= 0) {
+               struct ethtool_link_ksettings cmd;
+               int ethtool_ret;
+
+               /* Some devices may not be providing link settings */
+               ethtool_ret = __ethtool_get_link_ksettings(dev, &cmd);
+               if (!ethtool_ret)
+                       ret = cmd.base.speed;
+               else
+                       ret = ethtool_ret;
+       }
+       rtnl_unlock();
+
+       return ret;
+}
+
+static int ksocklnd_speed2cpp(int speed)
+{
+       /* Use the minimum of 1Gbps to avoid calling ilog2 with 0 */
+       if (speed < 1000)
+               speed = 1000;
+
+       /* Pick heuristically optimal conns_per_peer value
+        * for the specified ethernet interface speed (Mbps)
+        */
+       return ilog2(speed/1000) / 2 + 1;
+}
+#endif
+
+static int ksocklnd_lookup_conns_per_peer(struct lnet_ni *ni)
+{
+       int cpp = DEFAULT_CONNS_PER_PEER;
+#ifdef HAVE_ETHTOOL_LINK_SETTINGS
+       int speed = ksocklnd_ni_get_eth_intf_speed(ni);
+
+       CDEBUG(D_NET, "intf %s speed %d\n", ni->ni_interface, speed);
+       if (speed > 0)
+               cpp = ksocklnd_speed2cpp(speed);
+#endif
+       return cpp;
+}
+
 int ksocknal_tunables_init(void)
 {
        default_tunables.lnd_version = CURRENT_LND_VERSION;
@@ -285,6 +364,6 @@ void ksocknal_tunables_setup(struct lnet_ni *ni)
                        *ksocknal_tunables.ksnd_peerrtrcredits;
 
        if (!tunables->lnd_conns_per_peer)
-               tunables->lnd_conns_per_peer = (conns_per_peer) ?
-                       conns_per_peer : DEFAULT_CONNS_PER_PEER;
+               tunables->lnd_conns_per_peer =
+                       ksocklnd_lookup_conns_per_peer(ni);
 }