From 9a63a865e5f17827e533bf7ec2a667ca8eb82dd6 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Fri, 23 Sep 2022 15:20:51 -0700 Subject: [PATCH] LU-16051 o2iblnd: detect link state to set fatal error on ni To avoid selecting lnet ni which corresponds to a downed link for sending, add a mechanism for detecting ip-layer link events in o2iblnd. On ip link up/down events, find corresponding ni and toggle ni_fatal_error_on flag. This complements the existing mechanism for ib-layer link event handling. Lustre-change: https://review.whamcloud.com/48644 Lustre-commit: 30d73908087d5b2f0b18cce95826c4825c030ad4 Test-Parameters: trivial Signed-off-by: Serguei Smirnov Change-Id: I4720cd0a7bc577a522c7d40b54f821a4c12b670f Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49315 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Cyril Bordage Reviewed-by: Andreas Dilger --- lnet/klnds/o2iblnd/o2iblnd.c | 233 +++++++++++++++++++++++++++++++++++-------- lnet/klnds/o2iblnd/o2iblnd.h | 5 +- 2 files changed, 198 insertions(+), 40 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 3232dfe..cd3ba2f 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -2974,6 +2974,186 @@ kiblnd_destroy_dev(struct kib_dev *dev) LIBCFS_FREE(dev, sizeof(*dev)); } +static struct kib_dev * +kiblnd_dev_search(char *ifname) +{ + struct kib_dev *alias = NULL; + struct kib_dev *dev; + char *colon; + char *colon2; + + colon = strchr(ifname, ':'); + list_for_each_entry(dev, &kiblnd_data.kib_devs, ibd_list) { + if (strcmp(&dev->ibd_ifname[0], ifname) == 0) + return dev; + + if (alias != NULL) + continue; + + colon2 = strchr(dev->ibd_ifname, ':'); + if (colon != NULL) + *colon = 0; + if (colon2 != NULL) + *colon2 = 0; + + if (strcmp(&dev->ibd_ifname[0], ifname) == 0) + alias = dev; + + if (colon != NULL) + *colon = ':'; + if (colon2 != NULL) + *colon2 = ':'; + } + return alias; +} + +static int +kiblnd_handle_link_state_change(struct net_device *dev, + unsigned char operstate) +{ + struct lnet_ni *ni = NULL; + struct kib_dev *event_kibdev; + struct kib_net *net; + struct kib_net *cnxt; + bool link_down = !(operstate == IF_OPER_UP); + struct in_device *in_dev; + bool found_ip = false; + DECLARE_CONST_IN_IFADDR(ifa); + + event_kibdev = kiblnd_dev_search(dev->name); + + if (!event_kibdev) + goto out; + + list_for_each_entry_safe(net, cnxt, &event_kibdev->ibd_nets, ibn_list) { + found_ip = false; + + ni = net->ibn_ni; + + in_dev = __in_dev_get_rtnl(dev); + if (!in_dev) { + CDEBUG(D_NET, "Interface %s has no IPv4 status.\n", + dev->name); + CDEBUG(D_NET, "%s: set link fatal state to 1\n", + libcfs_nid2str(net->ibn_ni->ni_nid)); + atomic_set(&ni->ni_fatal_error_on, 1); + continue; + } + in_dev_for_each_ifa_rtnl(ifa, in_dev) { + if (htonl(event_kibdev->ibd_ifip) == ifa->ifa_local) + found_ip = true; + } + endfor_ifa(in_dev); + + if (!found_ip) { + CDEBUG(D_NET, "Interface %s has no matching ip\n", + dev->name); + CDEBUG(D_NET, "%s: set link fatal state to 1\n", + libcfs_nid2str(net->ibn_ni->ni_nid)); + atomic_set(&ni->ni_fatal_error_on, 1); + continue; + } + + if (link_down) { + CDEBUG(D_NET, "%s: set link fatal state to 1\n", + libcfs_nid2str(net->ibn_ni->ni_nid)); + atomic_set(&ni->ni_fatal_error_on, link_down); + } else { + CDEBUG(D_NET, "%s: set link fatal state to %u\n", + libcfs_nid2str(net->ibn_ni->ni_nid), + (kiblnd_get_link_status(dev) == 0)); + atomic_set(&ni->ni_fatal_error_on, + (kiblnd_get_link_status(dev) == 0)); + } + } +out: + return 0; +} + +static int +kiblnd_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event) +{ + struct kib_dev *event_kibdev; + struct kib_net *net; + struct kib_net *cnxt; + struct net_device *event_netdev = ifa->ifa_dev->dev; + + event_kibdev = kiblnd_dev_search(event_netdev->name); + + if (!event_kibdev) + goto out; + + if (htonl(event_kibdev->ibd_ifip) != ifa->ifa_local) + goto out; + + list_for_each_entry_safe(net, cnxt, &event_kibdev->ibd_nets, + ibn_list) { + CDEBUG(D_NET, "%s: set link fatal state to %u\n", + libcfs_nid2str(net->ibn_ni->ni_nid), + (event == NETDEV_DOWN)); + atomic_set(&net->ibn_ni->ni_fatal_error_on, + (event == NETDEV_DOWN)); + } +out: + return 0; +} + + +/************************************ + * Net device notifier event handler + ************************************/ +static int kiblnd_device_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + unsigned char operstate; + + operstate = dev->operstate; + + CDEBUG(D_NET, "devevent: status=%ld, iface=%s ifindex %d state %u\n", + event, dev->name, dev->ifindex, operstate); + + switch (event) { + case NETDEV_UP: + case NETDEV_DOWN: + case NETDEV_CHANGE: + kiblnd_handle_link_state_change(dev, operstate); + break; + } + + return NOTIFY_OK; +} + +/************************************ + * Inetaddr notifier event handler + ************************************/ +static int kiblnd_inetaddr_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct in_ifaddr *ifa = ptr; + + CDEBUG(D_NET, "addrevent: status %ld ip addr %pI4, netmask %pI4.\n", + event, &ifa->ifa_address, &ifa->ifa_mask); + + switch (event) { + case NETDEV_UP: + case NETDEV_DOWN: + case NETDEV_CHANGE: + kiblnd_handle_inetaddr_change(ifa, event); + break; + + } + return NOTIFY_OK; +} + +static struct notifier_block kiblnd_dev_notifier_block = { + .notifier_call = kiblnd_device_event, +}; + +static struct notifier_block kiblnd_inetaddr_notifier_block = { + .notifier_call = kiblnd_inetaddr_event, +}; + static void kiblnd_base_shutdown(void) { @@ -2985,9 +3165,14 @@ kiblnd_base_shutdown(void) CDEBUG(D_MALLOC, "before LND base cleanup: kmem %lld\n", libcfs_kmem_read()); - switch (kiblnd_data.kib_init) { - default: - LBUG(); + if (kiblnd_data.kib_init == IBLND_INIT_ALL) { + unregister_netdevice_notifier(&kiblnd_dev_notifier_block); + unregister_inetaddr_notifier(&kiblnd_inetaddr_notifier_block); + } + + switch (kiblnd_data.kib_init) { + default: + LBUG(); case IBLND_INIT_ALL: case IBLND_INIT_DATA: @@ -3192,9 +3377,12 @@ kiblnd_base_startup(struct net *ns) goto failed; } - /* flag everything initialised */ - kiblnd_data.kib_init = IBLND_INIT_ALL; - /*****************************************************/ + register_netdevice_notifier(&kiblnd_dev_notifier_block); + register_inetaddr_notifier(&kiblnd_inetaddr_notifier_block); + + /* flag everything initialised */ + kiblnd_data.kib_init = IBLND_INIT_ALL; + /*****************************************************/ return 0; @@ -3270,39 +3458,6 @@ static int kiblnd_dev_start_threads(struct kib_dev *dev, bool newdev, u32 *cpts, return 0; } -static struct kib_dev * -kiblnd_dev_search(char *ifname) -{ - struct kib_dev *alias = NULL; - struct kib_dev *dev; - char *colon; - char *colon2; - - colon = strchr(ifname, ':'); - list_for_each_entry(dev, &kiblnd_data.kib_devs, ibd_list) { - if (strcmp(&dev->ibd_ifname[0], ifname) == 0) - return dev; - - if (alias != NULL) - continue; - - colon2 = strchr(dev->ibd_ifname, ':'); - if (colon != NULL) - *colon = 0; - if (colon2 != NULL) - *colon2 = 0; - - if (strcmp(&dev->ibd_ifname[0], ifname) == 0) - alias = dev; - - if (colon != NULL) - *colon = ':'; - if (colon2 != NULL) - *colon2 = ':'; - } - return alias; -} - static int kiblnd_startup(struct lnet_ni *ni) { diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 9b185d7..a1bb47b 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -1278,4 +1278,7 @@ int kiblnd_recv(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg, unsigned int rlen); unsigned int kiblnd_get_dev_prio(struct lnet_ni *ni, unsigned int dev_idx); - +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) +#undef netdev_notifier_info_to_dev +#define netdev_notifier_info_to_dev(ndev) ndev +#endif -- 1.8.3.1