From e3d4371f7257339497566fa7bd28d8f53365dafc Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Tue, 11 Jul 2023 15:40:37 -0700 Subject: [PATCH] LU-16949 lnet: get monitor thread to update ping buffer Make sure that ping buffer updates requested by o2iblnd and socklnd are performed by the LNet monitor thread. Having the LNDs do these updates via an LNet API directly caused a lock-up due to spinlock acquisition while in an interrupt context in Centos 7.9 environment. To avoid LNet trying to update the ping buffer for an LNI which is still initializing, check that o2iblnd net is fully initialized (IBLND_INIT_ALL) before requesting the ping buffer update. Lustre-change: https://review.whamcloud.com/51635/ Lustre-commit: 7ac399c5aec01186ad4c9a7153aea400777c897f Fixes: da230373bd ("LU-16563 lnet: use discovered ni status") Signed-off-by: Serguei Smirnov Change-Id: I87ff8791937f5a0ead6096ff33e8c0a8087f8ddd Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51704 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Frank Sehr Reviewed-by: Andreas Dilger --- lnet/include/lnet/lib-lnet.h | 2 ++ lnet/include/lnet/lib-types.h | 3 +++ lnet/klnds/o2iblnd/o2iblnd.c | 15 +++++++++------ lnet/klnds/socklnd/socklnd.c | 4 ++-- lnet/lnet/api-ni.c | 13 +++++++++++-- lnet/lnet/lib-move.c | 5 ++++- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 33b32c2..f681088 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -1093,4 +1093,6 @@ lnet_set_route_aliveness(struct lnet_route *route, bool alive) alive ? "up" : "down"); } void lnet_update_ping_buffer(void); + +void lnet_mark_ping_buffer_for_update(void); #endif diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index f597a52..f4ebcfb 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -1187,6 +1187,9 @@ struct lnet { * work loops */ struct completion ln_started; + + /* for LNDs to signal that ping buffer needs updating */ + atomic_t ln_update_ping_buf; }; #endif diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index d2dc55f..ce4b41a 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -2567,12 +2567,13 @@ kiblnd_set_ni_fatal_on(struct kib_hca_dev *hdev, int val) if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - (val != ni_state_before)) + (val != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); } void @@ -3080,12 +3081,13 @@ kiblnd_handle_link_state_change(struct net_device *dev, ni_done: if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - (atomic_read(&ni->ni_fatal_error_on) != ni_state_before)) + (atomic_read(&ni->ni_fatal_error_on) != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } @@ -3117,12 +3119,13 @@ kiblnd_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event) ni_state_before = lnet_set_link_fatal_state(ni, link_down); if (!update_ping_buf && (ni->ni_state == LNET_NI_STATE_ACTIVE) && - ((event == NETDEV_DOWN) != ni_state_before)) + ((event == NETDEV_DOWN) != ni_state_before) && + (net->ibn_init == IBLND_INIT_ALL)) update_ping_buf = true; } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index bfa943e..63ef433c 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -1900,7 +1900,7 @@ ni_done: } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } @@ -1949,7 +1949,7 @@ ksocknal_handle_inetaddr_change(struct in_ifaddr *ifa, unsigned long event) } if (update_ping_buf) - lnet_update_ping_buffer(); + lnet_mark_ping_buffer_for_update(); out: return 0; } diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 46dcdee..0a48d34 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -3537,12 +3537,22 @@ out: return rc; } +void lnet_mark_ping_buffer_for_update(void) +{ + if (the_lnet.ln_routing) + return; + + atomic_set(&the_lnet.ln_update_ping_buf, 1); + complete(&the_lnet.ln_mt_wait_complete); +} +EXPORT_SYMBOL(lnet_mark_ping_buffer_for_update); + void lnet_update_ping_buffer(void) { struct lnet_ping_buffer *pbuf; struct lnet_handle_md ping_mdh; - if (the_lnet.ln_routing) + if (atomic_dec_if_positive(&the_lnet.ln_update_ping_buf) < 0) return; mutex_lock(&the_lnet.ln_api_mutex); @@ -3554,7 +3564,6 @@ void lnet_update_ping_buffer(void) mutex_unlock(&the_lnet.ln_api_mutex); } -EXPORT_SYMBOL(lnet_update_ping_buffer); void lnet_incr_dlc_seq(void) { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 2ae86af..34bc57b 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -3680,10 +3680,12 @@ lnet_monitor_thread(void *arg) * 1. Checks the aliveness of routers * 2. Checks if there are messages on the resend queue to resend * them. - * 3. Check if there are any NIs on the local recovery queue and + * 3. Checks if there are any NIs on the local recovery queue and * pings them * 4. Checks if there are any NIs on the remote recovery queue * and pings them. + * 5. Updates the ping buffer if requested by LNDs upon interface + * state change */ while (the_lnet.ln_mt_state == LNET_MT_STATE_RUNNING) { now = ktime_get_real_seconds(); @@ -3703,6 +3705,7 @@ lnet_monitor_thread(void *arg) lnet_recover_peer_nis(); recovery_timeout = now + lnet_recovery_interval; } + lnet_update_ping_buffer(); /* * TODO do we need to check if we should sleep without -- 1.8.3.1