From fc2df80e96dc5db9f3fb710893ccf6f442664471 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Tue, 8 Jun 2021 14:11:41 -0700 Subject: [PATCH] LU-14742 socklnd: detect link state to set fatal error on ni To help avoid selecting lnet ni which corresponds to a downed ethernet link for sending, add a mechanism for detecting link events in socklnd. On link up/down events, find corresponding ni and toggle ni_fatal_error_on flag, similar to o2iblnd way. Test-Parameters: trivial Signed-off-by: Serguei Smirnov Change-Id: Ie9f4f02fcb8b988c77bf63f751d5a621e79e9f58 Reviewed-on: https://review.whamcloud.com/43952 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Amir Shehata Reviewed-by: James Simmons Reviewed-by: Chris Horn Reviewed-by: Oleg Drokin --- lnet/klnds/socklnd/socklnd.c | 78 ++++++++++++++++++++++++++++++++++++++++++++ lnet/klnds/socklnd/socklnd.h | 6 ++++ 2 files changed, 84 insertions(+) diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index d9c428a..8e84644 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -1817,6 +1817,78 @@ ksocknal_free_buffers (void) } } +static int ksocknal_get_link_status(struct net_device *dev) +{ + int ret = -1; + + LASSERT(dev); + + if (!netif_running(dev)) + ret = 0; + /* Some devices may not be providing link settings */ + else if (dev->ethtool_ops->get_link) + ret = dev->ethtool_ops->get_link(dev); + + return ret; +} + +static int +ksocknal_handle_link_state_change(struct net_device *dev, + unsigned char operstate) +{ + struct lnet_ni *ni; + struct ksock_net *net; + struct ksock_net *cnxt; + int ifindex; + unsigned char link_down = !(operstate == IF_OPER_UP); + + ifindex = dev->ifindex; + + if (!ksocknal_data.ksnd_nnets) + goto out; + + list_for_each_entry_safe(net, cnxt, &ksocknal_data.ksnd_nets, + ksnn_list) { + if (net->ksnn_interface.ksni_index != ifindex) + continue; + ni = net->ksnn_ni; + if (link_down) + atomic_set(&ni->ni_fatal_error_on, link_down); + else + atomic_set(&ni->ni_fatal_error_on, + (ksocknal_get_link_status(dev) == 0)); + } +out: + return 0; +} + + +/************************************ + * Net device notifier event handler + ************************************/ +static int ksocknal_device_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + unsigned char operstate; + + operstate = dev->operstate; + + switch (event) { + case NETDEV_UP: + case NETDEV_DOWN: + case NETDEV_CHANGE: + ksocknal_handle_link_state_change(dev, operstate); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block ksocknal_notifier_block = { + .notifier_call = ksocknal_device_event, +}; + static void ksocknal_base_shutdown(void) { @@ -1828,6 +1900,9 @@ ksocknal_base_shutdown(void) libcfs_kmem_read()); LASSERT (ksocknal_data.ksnd_nnets == 0); + if (ksocknal_data.ksnd_init == SOCKNAL_INIT_ALL) + unregister_netdevice_notifier(&ksocknal_notifier_block); + switch (ksocknal_data.ksnd_init) { default: LASSERT(0); @@ -1994,6 +2069,8 @@ ksocknal_base_startup(void) goto failed; } + register_netdevice_notifier(&ksocknal_notifier_block); + /* flag everything initialised */ ksocknal_data.ksnd_init = SOCKNAL_INIT_ALL; @@ -2276,6 +2353,7 @@ ksocknal_startup(struct lnet_ni *ni) ntohl(((struct sockaddr_in *) &ksi->ksni_addr)->sin_addr.s_addr)); list_add(&net->ksnn_list, &ksocknal_data.ksnd_nets); + net->ksnn_ni = ni; ksocknal_data.ksnd_nnets++; return 0; diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 08bbab9..f5c29d5 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -168,6 +168,7 @@ struct ksock_net { struct list_head ksnn_list; /* chain on global list */ atomic_t ksnn_npeers; /* # peers */ struct ksock_interface ksnn_interface; /* IP interface */ + struct lnet_ni *ksnn_ni; }; /* When the ksock_net is shut down, this (negative) bias is added to * ksnn_npeers, which prevents new peers from being added. @@ -455,6 +456,11 @@ extern const struct ksock_proto ksocknal_protocol_v3x; #define CPU_MASK_NONE 0UL #endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) +#undef netdev_notifier_info_to_dev +#define netdev_notifier_info_to_dev(ndev) ndev +#endif + static inline __u32 ksocknal_csum(__u32 crc, unsigned char const *p, size_t len) { #if 1 -- 1.8.3.1