From c44afcfb72a1c2fd8392bfab3143c3835b146be6 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Wed, 28 Jul 2021 14:47:39 -0700 Subject: [PATCH] LU-12815 socklnd: set conns_per_peer based on link speed Specifying conns_per_peer=0 for a ni is now used to set the conns_per_peer as a function of the corresponding link speed as follows: conns_per_peer = (ilog2(Gbps) / 2 + 1) Listed below are the resulting defaults for common link speeds: 100Gbps, 200Gbps -> 4 50Gbps -> 3 5Gbps, 10Gbps -> 2 less than 4Gbps -> 1 Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Serguei Smirnov Change-Id: Ief2b33a796c180d8669bd5796b3e35ec748423a5 Reviewed-on: https://review.whamcloud.com/44417 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lnet/autoconf/lustre-lnet.m4 | 11 +++++ lnet/klnds/socklnd/socklnd.c | 1 - lnet/klnds/socklnd/socklnd_modparams.c | 83 +++++++++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index cd0bc82..c6be763 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -570,6 +570,17 @@ AS_IF([test $ENABLEO2IB != "no"], [ [rdma_connect_locked is defined]) ]) + # ethtool_link_settings was added in Linux 4.6 + LB_CHECK_COMPILE([if 'ethtool_link_settings' exists], + ethtool_link_settings, [ + #include + ],[ + struct ethtool_link_ksettings cmd; + ],[ + AC_DEFINE(HAVE_ETHTOOL_LINK_SETTINGS, 1, + [ethtool_link_settings is defined]) + ]) + EXTRA_CHECK_INCLUDE="" AC_DEFUN([LN_CONFIG_O2IB_SRC], []) AC_DEFUN([LN_CONFIG_O2IB_RESULTS], []) diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index ab4adb4..ec7e069 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -2367,7 +2367,6 @@ fail_0: return -ENETDOWN; } - static void __exit ksocklnd_exit(void) { lnet_unregister_lnd(&the_ksocklnd); diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c index 72ae1ac..967575a 100644 --- a/lnet/klnds/socklnd/socklnd_modparams.c +++ b/lnet/klnds/socklnd/socklnd_modparams.c @@ -25,6 +25,10 @@ #if defined(__x86_64__) || defined(__i386__) #include #endif +#ifdef HAVE_ETHTOOL_LINK_SETTINGS +#include +#include +#endif #define CURRENT_LND_VERSION 1 @@ -178,6 +182,81 @@ static inline bool is_native_host(void) struct ksock_tunables ksocknal_tunables; static struct lnet_ioctl_config_socklnd_tunables default_tunables; +#ifdef HAVE_ETHTOOL_LINK_SETTINGS +static int ksocklnd_ni_get_eth_intf_speed(struct lnet_ni *ni) +{ + struct net_device *dev; + int intf_idx = -1; + int ret = -1; + + DECLARE_CONST_IN_IFADDR(ifa); + + rtnl_lock(); + for_each_netdev(ni->ni_net_ns, dev) { + int flags = dev_get_flags(dev); + struct in_device *in_dev; + + if (flags & IFF_LOOPBACK) /* skip the loopback IF */ + continue; + + if (!(flags & IFF_UP)) + continue; + + in_dev = __in_dev_get_rcu(dev); + if (!in_dev) + continue; + + in_dev_for_each_ifa_rcu(ifa, in_dev) { + if (strcmp(ifa->ifa_label, ni->ni_interface) == 0) + intf_idx = dev->ifindex; + } + endfor_ifa(in_dev); + + if (intf_idx >= 0) + break; + } + if (intf_idx >= 0) { + struct ethtool_link_ksettings cmd; + int ethtool_ret; + + /* Some devices may not be providing link settings */ + ethtool_ret = __ethtool_get_link_ksettings(dev, &cmd); + if (!ethtool_ret) + ret = cmd.base.speed; + else + ret = ethtool_ret; + } + rtnl_unlock(); + + return ret; +} + +static int ksocklnd_speed2cpp(int speed) +{ + /* Use the minimum of 1Gbps to avoid calling ilog2 with 0 */ + if (speed < 1000) + speed = 1000; + + /* Pick heuristically optimal conns_per_peer value + * for the specified ethernet interface speed (Mbps) + */ + return ilog2(speed/1000) / 2 + 1; +} +#endif + +static int ksocklnd_lookup_conns_per_peer(struct lnet_ni *ni) +{ + int cpp = DEFAULT_CONNS_PER_PEER; +#ifdef HAVE_ETHTOOL_LINK_SETTINGS + int speed = ksocklnd_ni_get_eth_intf_speed(ni); + + CDEBUG(D_NET, "intf %s speed %d\n", ni->ni_interface, speed); + if (speed > 0) + cpp = ksocklnd_speed2cpp(speed); +#endif + return cpp; +} + int ksocknal_tunables_init(void) { default_tunables.lnd_version = CURRENT_LND_VERSION; @@ -285,6 +364,6 @@ void ksocknal_tunables_setup(struct lnet_ni *ni) *ksocknal_tunables.ksnd_peerrtrcredits; if (!tunables->lnd_conns_per_peer) - tunables->lnd_conns_per_peer = (conns_per_peer) ? - conns_per_peer : DEFAULT_CONNS_PER_PEER; + tunables->lnd_conns_per_peer = + ksocklnd_lookup_conns_per_peer(ni); } -- 1.8.3.1