From 1ea6c87d415144522a1c50c08063c9d30d585325 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Wed, 27 Oct 2021 20:10:17 +0000 Subject: [PATCH] LU-15595 lnet: Always use ping reply to set route lr_alive We currently process discovery ping replies in different ways depending on whether the gateway has discovery enabled or disabled (or the local peer doing the processing has discovery enabled or disabled). When DD is disabled we process the ping reply to set the lr_alive field of lnet_route because the peer objects for non-MR routers do not contain all the information needed to calculate the route aliveness when a message is being sent. When DD is enabled then we don't do any special processing of the ping reply. We simply let discovery update the NI status for the GW's peer NIs and then we calculate the route aliveness on every send. We issue discovery pings to routers every alive_router_check_interval seconds (default 60), but we calculate route aliveness on every send to a remote network (1000s of times per seconds). Thus, it is better to slightly duplicate the effort expended when we receive a discovery reply so that we can avoid calculating route aliveness on every send. Since both lr_alive and hop type are being set on each ping reply, for both DD enabled and disabled cases, we can remove the code for updating lr_alive and hop type from lnet_router_discovery_complete(). If discover encounters a fatal error, we still set the status of each peer NI, as well as all routes, to down in lnet_router_discovery_complete(). Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Chris Horn Change-Id: If4838c269a89885ba3763f62847e294804edf62e Reviewed-on: https://review.whamcloud.com/46624 Reviewed-by: Cyril Bordage Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andriy Skulysh Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-lnet.h | 3 +- lnet/lnet/peer.c | 15 ++++----- lnet/lnet/router.c | 73 +++++++++----------------------------------- 3 files changed, 23 insertions(+), 68 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 06854ad..0974c95 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -925,7 +925,8 @@ int lnet_peer_buffer_credits(struct lnet_net *net); void lnet_consolidate_routes_locked(struct lnet_peer *orig_lp, struct lnet_peer *new_lp); void lnet_router_discovery_complete(struct lnet_peer *lp); -void lnet_router_discovery_ping_reply(struct lnet_peer *lp); +void lnet_router_discovery_ping_reply(struct lnet_peer *lp, + struct lnet_ping_buffer *pbuf); int lnet_monitor_thr_start(void); void lnet_monitor_thr_stop(void); diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index 11c22f0..1fcb697 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -2760,15 +2760,6 @@ lnet_discovery_event_reply(struct lnet_peer *lp, struct lnet_event *ev) out: lp->lp_state &= ~LNET_PEER_PING_SENT; spin_unlock(&lp->lp_lock); - - lnet_net_lock(LNET_LOCK_EX); - /* - * If this peer is a gateway, call the routing callback to - * handle the ping reply - */ - if (lp->lp_rtr_refcount > 0) - lnet_router_discovery_ping_reply(lp); - lnet_net_unlock(LNET_LOCK_EX); } /* @@ -3070,6 +3061,12 @@ static int lnet_peer_merge_data(struct lnet_peer *lp, */ rc = 0; out: + /* If this peer is a gateway, invoke the routing callback to update + * the associated route status + */ + if (lp->lp_rtr_refcount > 0) + lnet_router_discovery_ping_reply(lp, pbuf); + CFS_FREE_PTR_ARRAY(curnis, nnis); CFS_FREE_PTR_ARRAY(addnis, nnis); CFS_FREE_PTR_ARRAY(delnis, nnis); diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index 93ae181..3960785 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -158,6 +158,7 @@ rtr_sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp) void lnet_move_route(struct lnet_route *route, struct lnet_peer *lp, struct list_head *rt_list) +__must_hold(&the_lnet.ln_api_mutex) { struct lnet_remotenet *rnet; struct list_head zombies; @@ -391,62 +392,31 @@ lnet_check_route_inconsistency(struct lnet_route *route) } } -static void -lnet_set_route_hop_type(struct lnet_peer *gw, struct lnet_route *route) -{ - struct lnet_peer_net *lpn; - bool single_hop = false; - - list_for_each_entry(lpn, &gw->lp_peer_nets, lpn_peer_nets) { - if (route->lr_net == lpn->lpn_net_id) { - single_hop = true; - break; - } - } - route->lr_single_hop = single_hop; - lnet_check_route_inconsistency(route); -} - -/* Must hold net_lock/EX */ +/* Routes are added and removed under both ln_api_mutex and net_lock/EX + * Since we are not modifying anything we simply require the ln_api_mutex be + * held so that things are not modified underneath us + */ void -lnet_router_discovery_ping_reply(struct lnet_peer *lp) +lnet_router_discovery_ping_reply(struct lnet_peer *lp, + struct lnet_ping_buffer *pbuf) +__must_hold(&the_lnet.ln_api_mutex) { - struct lnet_ping_buffer *pbuf = lp->lp_data; struct lnet_peer_net *llpn; struct lnet_route *route; bool single_hop = false; bool net_up = false; - unsigned lp_state; __u32 net; int i; - - spin_lock(&lp->lp_lock); - lp_state = lp->lp_state; - - /* only handle replies if discovery is disabled. */ - if (!lnet_is_discovery_disabled_locked(lp)) { - spin_unlock(&lp->lp_lock); - return; - } - - spin_unlock(&lp->lp_lock); - - if (lp_state & LNET_PEER_PING_FAILED || - pbuf->pb_info.pi_features & LNET_PING_FEAT_RTE_DISABLED) { - CDEBUG(D_NET, "Set routes down for gw %s because %s %d\n", - libcfs_nidstr(&lp->lp_primary_nid), - lp_state & LNET_PEER_PING_FAILED ? "ping failed" : - "route feature is disabled", lp->lp_ping_error); - /* If the ping failed or the peer has routing disabled then - * mark the routes served by this peer down - */ + if (pbuf->pb_info.pi_features & LNET_PING_FEAT_RTE_DISABLED) { + CERROR("Peer %s is being used as a gateway but routing feature is not turned on\n", + libcfs_nidstr(&lp->lp_primary_nid)); list_for_each_entry(route, &lp->lp_routes, lr_gwlist) lnet_set_route_aliveness(route, false); return; } - CDEBUG(D_NET, "Discovery is disabled. Processing reply for gw: %s:%d\n", + CDEBUG(D_NET, "Processing reply for gw: %s nnis %d\n", libcfs_nidstr(&lp->lp_primary_nid), pbuf->pb_info.pi_nnis); /* @@ -510,23 +480,8 @@ lnet_router_discovery_complete(struct lnet_peer *lp) lp->lp_alive = lp->lp_dc_error == 0; spin_unlock(&lp->lp_lock); - if (!lp->lp_dc_error) { - /* ping replies are being handled when discovery is disabled */ - if (lnet_is_discovery_disabled_locked(lp)) - return; - - /* - * mark single-hop routes. If the remote net is not configured on - * the gateway we assume this is intentional and we mark the - * gateway as multi-hop - */ - list_for_each_entry(route, &lp->lp_routes, lr_gwlist) { - lnet_set_route_aliveness(route, true); - lnet_set_route_hop_type(lp, route); - } - + if (!lp->lp_dc_error) return; - } /* * We do not send messages directly to the remote interfaces @@ -660,6 +615,7 @@ lnet_add_route_to_rnet(struct lnet_remotenet *rnet, struct lnet_route *route) int lnet_add_route(__u32 net, __u32 hops, struct lnet_nid *gateway, __u32 priority, __u32 sensitivity) +__must_hold(&the_lnet.ln_api_mutex) { struct list_head *route_entry; struct lnet_remotenet *rnet; @@ -845,6 +801,7 @@ lnet_del_route_from_rnet(struct lnet_nid *gw_nid, int lnet_del_route(__u32 net, struct lnet_nid *gw) +__must_hold(&the_lnet.ln_api_mutex) { LIST_HEAD(rnet_zombies); struct lnet_remotenet *rnet; -- 1.8.3.1