From 7ac399c5aec01186ad4c9a7153aea400777c897f Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Tue, 11 Jul 2023 15:40:37 -0700 Subject: [PATCH] LU-16949 lnet: get monitor thread to update ping buffer Make sure that ping buffer updates requested by o2iblnd and socklnd are performed by the LNet monitor thread. Having the LNDs do these updates via an LNet API directly caused a lock-up due to spinlock acquisition while in an interrupt context in Centos 7.9 environment. To avoid LNet trying to update the ping buffer for an LNI which is still initializing, check that o2iblnd net is fully initialized (IBLND_INIT_ALL) before requesting the ping buffer update. Fixes: da230373bd ("LU-16563 lnet: use discovered ni status") Signed-off-by: Serguei Smirnov Change-Id: I87ff8791937f5a0ead6096ff33e8c0a8087f8ddd Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51635 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Cyril Bordage Reviewed-by: Chris Horn Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-lnet.h | 1 + lnet/include/lnet/lib-types.h | 3 +++ lnet/klnds/o2iblnd/o2iblnd.c | 15 +++++++++------ lnet/klnds/socklnd/socklnd.c | 4 ++-- lnet/lnet/api-ni.c | 13 +++++++++++-- lnet/lnet/lib-move.c | 5 ++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 9e36659..3fcf86e 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -1315,4 +1315,5 @@ static inline int notifier_from_ioctl_errno(int err) return notifier_from_errno(err) | NOTIFY_STOP_MASK; } +void lnet_mark_ping_buffer_for_update(void); #endif diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 8cf687f..52470b3 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -1519,6 +1519,9 @@ struct lnet { atomic_t ln_late_msg_count; /* Total amount of time past their deadline for all late ^ messages */ atomic64_t ln_late_msg_nsecs; + + /* for LNDs to signal that ping buffer needs updating */ + atomic_t ln_update_ping_buf; }; struct genl_filter_list { diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 6fca85a..818b900 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -2649,12 +2649,13 @@ kiblnd_set_ni_fatal_on(struct kib_hca_dev *hdev, int val) if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - (val != ni_state_before)) + (val != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); } static void @@ -3154,12 +3155,13 @@ kiblnd_handle_link_state_change(struct net_device *dev, ni_done: if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - (atomic_read(&ni->ni_fatal_error_on) != ni_state_before)) + (atomic_read(&ni->ni_fatal_error_on) != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } @@ -3191,12 +3193,13 @@ kiblnd_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event) ni_state_before = lnet_set_link_fatal_state(ni, link_down); if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - ((event == NETDEV_DOWN) != ni_state_before)) + ((event == NETDEV_DOWN) != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index bf340e3..d8e1e84 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -1999,7 +1999,7 @@ ni_done: } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } @@ -2048,7 +2048,7 @@ ksocknal_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event) } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index ae59d5f..15c60fa 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -3920,12 +3920,22 @@ out: return rc; } +void lnet_mark_ping_buffer_for_update(void) +{ + if (the_lnet.ln_routing) + return; + + atomic_set(&the_lnet.ln_update_ping_buf, 1); + complete(&the_lnet.ln_mt_wait_complete); +} +EXPORT_SYMBOL(lnet_mark_ping_buffer_for_update); + void lnet_update_ping_buffer(void) { struct lnet_ping_buffer *pbuf; struct lnet_handle_md ping_mdh; - if (the_lnet.ln_routing) + if (atomic_dec_if_positive(&the_lnet.ln_update_ping_buf) < 0) return; mutex_lock(&the_lnet.ln_api_mutex); @@ -3938,7 +3948,6 @@ void lnet_update_ping_buffer(void) mutex_unlock(&the_lnet.ln_api_mutex); } -EXPORT_SYMBOL(lnet_update_ping_buffer); void lnet_incr_dlc_seq(void) { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index a880e1e..ab9cbfe 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -4039,10 +4039,12 @@ lnet_monitor_thread(void *arg) * 1. Checks the aliveness of routers * 2. Checks if there are messages on the resend queue to resend * them. - * 3. Check if there are any NIs on the local recovery queue and + * 3. Checks if there are any NIs on the local recovery queue and * pings them * 4. Checks if there are any NIs on the remote recovery queue * and pings them. + * 5. Updates the ping buffer if requested by LNDs upon interface + * state change */ while (the_lnet.ln_mt_state == LNET_MT_STATE_RUNNING) { now = ktime_get_real_seconds(); @@ -4061,6 +4063,7 @@ lnet_monitor_thread(void *arg) nlpnis = lnet_recover_peer_nis(peer_nids, LNET_MAX_NNIDS); lnet_health_update_console(local_nids, nnis, peer_nids, nlpnis, now); + lnet_update_ping_buffer(); /* * TODO do we need to check if we should sleep without -- 1.8.3.1