From 51a5b4df5bbbf5fd12c73d2722b230e93fe93327 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Tue, 29 May 2012 14:15:39 +0800 Subject: [PATCH] LU-56 lnet: parse RC ping in event callback In current LNet, we don't parse router-checker ping in EQ callback, NI information from remote side is attached on peer and it's parsed for each sending. This is OK for now but with upcoming SMP improvements, the global LNET_LOCK is removed and EQ callback and peer/router are proteced by different locks, which means sink buffer of RC ping could be in progressing of change when other thread is reading it in lnet_send(), and get wrong information. This patch will parse ping information in EQ callback, and store it in a new member lnet_route_t::lr_downis which is just a integer, although reader in lnet_send() still don't have lock while reading it, but it's OK because it's just an integer and reader can atomically read reliable result even w/o any lock. It should also be a little faster than original implememtation because we don't have to parse RC ping, which should be updated once for many seconds, for each sending. Signed-off-by: Liang Zhen Change-Id: I5b89fabfd22f22ba171ef63d30da2d32e2788b16 Reviewed-on: http://review.whamcloud.com/2932 Reviewed-by: Doug Oucharek Tested-by: Hudson Tested-by: Maloo Reviewed-by: Bobi Jam Reviewed-by: Andreas Dilger --- lnet/include/lnet/lib-lnet.h | 1 - lnet/include/lnet/lib-types.h | 30 ++-- lnet/lnet/lib-move.c | 89 ++++++----- lnet/lnet/peer.c | 6 +- lnet/lnet/router.c | 338 +++++++++++++++++++++++------------------- lnet/lnet/router_proc.c | 16 +- 6 files changed, 275 insertions(+), 205 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index f5a49b7..ec3bd41 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -809,7 +809,6 @@ int lnet_peer_buffer_credits(lnet_ni_t *ni); int lnet_router_checker_start(void); void lnet_router_checker_stop(void); void lnet_swap_pinginfo(lnet_ping_info_t *info); -int lnet_router_down_ni(lnet_peer_t *rtr, __u32 net); int lnet_ping_target_init(void); void lnet_ping_target_fini(void); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 514e099..8febd47 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -407,8 +407,12 @@ typedef struct lnet_ni { } lnet_ni_t; #define LNET_PROTO_PING_MATCHBITS 0x8000000000000000LL -#define LNET_PROTO_PING_VERSION 2 -#define LNET_PROTO_PING_VERSION1 1 +enum { + LNET_PROTO_PING_UNKNOWN = 0, /* unknown */ + LNET_PROTO_PING_VERSION_1 = 1, /* old version */ + LNET_PROTO_PING_VERSION = 2, /* current version */ +}; + typedef struct { __u32 pi_magic; __u32 pi_version; @@ -421,9 +425,11 @@ typedef struct { #define LNET_MAX_RTR_NIS 16 #define LNET_PINGINFO_SIZE offsetof(lnet_ping_info_t, pi_ni[LNET_MAX_RTR_NIS]) typedef struct { - cfs_list_t rcd_list; /* chain on the_lnet.ln_zombie_rcd */ - lnet_handle_md_t rcd_mdh; /* ping buffer MD */ - lnet_ping_info_t *rcd_pinginfo; /* ping buffer */ + /* chain on the_lnet.ln_zombie_rcd or ln_deathrow_rcd */ + cfs_list_t rcd_list; + lnet_handle_md_t rcd_mdh; /* ping buffer MD */ + struct lnet_peer *rcd_gateway; /* reference to gateway */ + lnet_ping_info_t *rcd_pinginfo; /* ping buffer */ } lnet_rc_data_t; typedef struct lnet_peer { @@ -451,7 +457,10 @@ typedef struct lnet_peer { lnet_nid_t lp_nid; /* peer's NID */ int lp_refcount; /* # refs */ int lp_rtr_refcount; /* # refs from lnet_route_t::lr_gateway */ - lnet_rc_data_t *lp_rcd; /* router checker state */ + /* returned RC ping version */ + unsigned int lp_ping_version; + cfs_list_t lp_routes; /* routers on this peer */ + lnet_rc_data_t *lp_rcd; /* router checker state */ } lnet_peer_t; @@ -470,9 +479,12 @@ struct lnet_peer_table { #define lnet_peer_aliveness_enabled(lp) ((lp)->lp_ni->ni_peertimeout > 0) typedef struct { - cfs_list_t lr_list; /* chain on net */ - lnet_peer_t *lr_gateway; /* router node */ - unsigned int lr_hops; /* how far I am */ + cfs_list_t lr_list; /* chain on net */ + cfs_list_t lr_gwlist; /* chain on gateway */ + lnet_peer_t *lr_gateway; /* router node */ + __u32 lr_net; /* remote network number */ + unsigned int lr_downis; /* number of down NIs */ + unsigned int lr_hops; /* how far I am */ } lnet_route_t; typedef struct { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 683d555..c760a92 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1146,18 +1146,65 @@ lnet_return_credits_locked (lnet_msg_t *msg) } } +static lnet_peer_t * +lnet_find_route_locked(lnet_ni_t *ni, lnet_nid_t target) +{ + lnet_remotenet_t *rnet; + lnet_route_t *rtr; + lnet_route_t *rtr_best; + lnet_route_t *rtr_last; + struct lnet_peer *lp_best; + struct lnet_peer *lp; + int rc; + + rnet = lnet_find_net_locked(LNET_NIDNET(target)); + if (rnet == NULL) + return NULL; + + lp_best = NULL; + rtr_best = rtr_last = NULL; + cfs_list_for_each_entry(rtr, &rnet->lrn_routes, lr_list) { + lp = rtr->lr_gateway; + + if (!lp->lp_alive || /* gateway is down */ + (lp->lp_ping_version == LNET_PROTO_PING_VERSION && + rtr->lr_downis != 0)) /* NI to target is down */ + continue; + + if (ni != NULL && lp->lp_ni != ni) + continue; + + if (lp_best == NULL) { + rtr_best = rtr_last = rtr; + lp_best = lp; + continue; + } + + rc = lnet_compare_routes(rtr, rtr_best); + if (rc < 0) + continue; + + rtr_best = rtr; + lp_best = lp; + } + + if (rtr_best != NULL) { + /* Place selected route at the end of the route list to ensure + * fairness; everything else being equal... */ + cfs_list_del(&rtr_best->lr_list); + cfs_list_add_tail(&rtr_best->lr_list, &rnet->lrn_routes); + } + + return lp_best; +} + int lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) { lnet_nid_t dst_nid = msg->msg_target.nid; lnet_ni_t *src_ni; lnet_ni_t *local_ni; - lnet_remotenet_t *rnet; - lnet_route_t *route; - lnet_route_t *best_route; - cfs_list_t *tmp; lnet_peer_t *lp; - lnet_peer_t *lp2; int rc; LASSERT (msg->msg_txpeer == NULL); @@ -1247,33 +1294,7 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) LNET_LOCK(); #endif /* sending to a remote network */ - rnet = lnet_find_net_locked(LNET_NIDNET(dst_nid)); - if (rnet == NULL) { - if (src_ni != NULL) - lnet_ni_decref_locked(src_ni); - LNET_UNLOCK(); - LCONSOLE_WARN("No route to %s\n", - libcfs_id2str(msg->msg_target)); - return -EHOSTUNREACH; - } - - /* Find the best gateway I can use */ - lp = NULL; - best_route = NULL; - cfs_list_for_each(tmp, &rnet->lrn_routes) { - route = cfs_list_entry(tmp, lnet_route_t, lr_list); - lp2 = route->lr_gateway; - - if (lp2->lp_alive && - lnet_router_down_ni(lp2, rnet->lrn_net) <= 0 && - (src_ni == NULL || lp2->lp_ni == src_ni) && - (lp == NULL || - lnet_compare_routes(route, best_route) > 0)) { - best_route = route; - lp = lp2; - } - } - + lp = lnet_find_route_locked(src_ni, dst_nid); if (lp == NULL) { if (src_ni != NULL) lnet_ni_decref_locked(src_ni); @@ -1286,10 +1307,6 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) return -EHOSTUNREACH; } - /* Place selected route at the end of the route list to ensure - * fairness; everything else being equal... */ - cfs_list_del(&best_route->lr_list); - cfs_list_add_tail(&best_route->lr_list, &rnet->lrn_routes); CDEBUG(D_NET, "Best route to %s via %s for %s %d\n", libcfs_nid2str(dst_nid), libcfs_nid2str(lp->lp_nid), lnet_msgtyp2str(msg->msg_type), msg->msg_len); diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index 51123f0..419372f 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -231,11 +231,10 @@ lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid) return -ENOMEM; } - memset(lp, 0, sizeof(*lp)); /* zero counters etc */ - CFS_INIT_LIST_HEAD(&lp->lp_txq); CFS_INIT_LIST_HEAD(&lp->lp_rtrq); - + CFS_INIT_LIST_HEAD(&lp->lp_routes); + lp->lp_notify = 0; lp->lp_notifylnd = 0; lp->lp_notifying = 0; @@ -245,6 +244,7 @@ lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid) lp->lp_last_alive = cfs_time_current(); /* assumes alive */ lp->lp_last_query = 0; /* haven't asked NI yet */ lp->lp_ping_timestamp = 0; + lp->lp_ping_version = LNET_PROTO_PING_UNKNOWN; lp->lp_nid = nid; lp->lp_refcount = 2; /* 1 for caller; 1 for hash */ lp->lp_rtr_refcount = 0; diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index ef1ff34..08bb57c 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -122,6 +122,8 @@ lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, cfs_time_t when) lp->lp_alive = !(!alive); /* 1 bit! */ lp->lp_notify = 1; lp->lp_notifylnd |= notifylnd; + if (lp->lp_alive) + lp->lp_ping_version = LNET_PROTO_PING_UNKNOWN; /* reset */ CDEBUG(D_NET, "set %s %d\n", libcfs_nid2str(lp->lp_nid), alive); } @@ -198,6 +200,8 @@ lnet_rtr_decref_locked(lnet_peer_t *lp) lp->lp_rtr_refcount--; if (lp->lp_rtr_refcount == 0) { + LASSERT(cfs_list_empty(&lp->lp_routes)); + if (lp->lp_rcd != NULL) { cfs_list_add(&lp->lp_rcd->rcd_list, &the_lnet.ln_rcd_deathrow); @@ -279,6 +283,7 @@ lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route) offset--; } cfs_list_add(&route->lr_list, e); + cfs_list_add(&route->lr_gwlist, &route->lr_gateway->lp_routes); the_lnet.ln_remote_nets_version++; lnet_rtr_addref_locked(route->lr_gateway); @@ -325,6 +330,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) CFS_INIT_LIST_HEAD(&rnet->lrn_routes); rnet->lrn_net = net; route->lr_hops = hops; + route->lr_net = net; LNET_LOCK(); @@ -335,11 +341,13 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) LIBCFS_FREE(route, sizeof(*route)); LIBCFS_FREE(rnet, sizeof(*rnet)); - if (rc == -EHOSTUNREACH) /* gateway is not on a local net */ + if (rc == -EHOSTUNREACH) { /* gateway is not on a local net */ return 0; /* ignore the route entry */ - - CERROR("Error %d creating route %s %d %s\n", rc, - libcfs_net2str(net), hops, libcfs_nid2str(gateway)); + } else { + CERROR("Error %d creating route %s %d %s\n", rc, + libcfs_net2str(net), hops, + libcfs_nid2str(gateway)); + } return rc; } @@ -367,22 +375,25 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) } if (add_route) { - ni = route->lr_gateway->lp_ni; - lnet_ni_addref_locked(ni); + lnet_peer_addref_locked(route->lr_gateway); /* +1 for notify */ + lnet_add_route_to_rnet(rnet2, route); - lnet_add_route_to_rnet(rnet2, route); - LNET_UNLOCK(); + ni = route->lr_gateway->lp_ni; + LNET_UNLOCK(); - /* XXX Assume alive */ - if (ni->ni_lnd->lnd_notify != NULL) - (ni->ni_lnd->lnd_notify)(ni, gateway, 1); + /* XXX Assume alive */ + if (ni->ni_lnd->lnd_notify != NULL) + (ni->ni_lnd->lnd_notify)(ni, gateway, 1); - lnet_ni_decref(ni); - } else { - lnet_peer_decref_locked(route->lr_gateway); - LNET_UNLOCK(); - LIBCFS_FREE(route, sizeof(*route)); - } + LNET_LOCK(); + } + + /* -1 for notify or !add_route */ + lnet_peer_decref_locked(route->lr_gateway); + LNET_UNLOCK(); + + if (!add_route) + LIBCFS_FREE(route, sizeof(*route)); if (rnet != rnet2) LIBCFS_FREE(rnet, sizeof(*rnet)); @@ -406,20 +417,31 @@ lnet_check_routes (void) route2 = NULL; cfs_list_for_each (e2, &rnet->lrn_routes) { + lnet_nid_t nid1; + lnet_nid_t nid2; + int net; + route = cfs_list_entry(e2, lnet_route_t, lr_list); - if (route2 == NULL) - route2 = route; - else if (route->lr_gateway->lp_ni != - route2->lr_gateway->lp_ni) { - LNET_UNLOCK(); + if (route2 == NULL) { + route2 = route; + continue; + } - CERROR("Routes to %s via %s and %s not supported\n", - libcfs_net2str(rnet->lrn_net), - libcfs_nid2str(route->lr_gateway->lp_nid), - libcfs_nid2str(route2->lr_gateway->lp_nid)); - return -EINVAL; - } + if (route->lr_gateway->lp_ni == + route2->lr_gateway->lp_ni) + continue; + + nid1 = route->lr_gateway->lp_nid; + nid2 = route2->lr_gateway->lp_nid; + net = rnet->lrn_net; + + LNET_UNLOCK(); + + CERROR("Routes to %s via %s and %s not supported\n", + libcfs_net2str(net), libcfs_nid2str(nid1), + libcfs_nid2str(nid2)); + return -EINVAL; } } @@ -430,6 +452,7 @@ lnet_check_routes (void) int lnet_del_route (__u32 net, lnet_nid_t gw_nid) { + struct lnet_peer *gateway; lnet_remotenet_t *rnet; lnet_route_t *route; cfs_list_t *e1; @@ -455,11 +478,13 @@ lnet_del_route (__u32 net, lnet_nid_t gw_nid) cfs_list_for_each (e2, &rnet->lrn_routes) { route = cfs_list_entry(e2, lnet_route_t, lr_list); - if (!(gw_nid == LNET_NID_ANY || - gw_nid == route->lr_gateway->lp_nid)) - continue; + gateway = route->lr_gateway; + if (!(gw_nid == LNET_NID_ANY || + gw_nid == gateway->lp_nid)) + continue; - cfs_list_del(&route->lr_list); + cfs_list_del(&route->lr_list); + cfs_list_del(&route->lr_gwlist); the_lnet.ln_remote_nets_version++; if (cfs_list_empty(&rnet->lrn_routes)) @@ -467,8 +492,8 @@ lnet_del_route (__u32 net, lnet_nid_t gw_nid) else rnet = NULL; - lnet_rtr_decref_locked(route->lr_gateway); - lnet_peer_decref_locked(route->lr_gateway); + lnet_rtr_decref_locked(gateway); + lnet_peer_decref_locked(gateway); LNET_UNLOCK(); LIBCFS_FREE(route, sizeof (*route)); @@ -529,6 +554,7 @@ lnet_swap_pinginfo(lnet_ping_info_t *info) int i; lnet_ni_status_t *stat; + __swab32s(&info->pi_magic); __swab32s(&info->pi_version); __swab32s(&info->pi_pid); __swab32s(&info->pi_nnis); @@ -540,84 +566,139 @@ lnet_swap_pinginfo(lnet_ping_info_t *info) return; } -/* Returns # of down NIs, or negative error codes; ignore downed NIs - * if a NI in 'net' is up */ -int -lnet_router_down_ni(lnet_peer_t *rtr, __u32 net) +/** + * parse router-checker pinginfo, record number of down NIs for remote + * networks on that router. + */ +static void +lnet_parse_rc_info(lnet_rc_data_t *rcd) { - int i; - int down = 0; - int ptl_up = 0; - int ptl_down = 0; - lnet_ping_info_t *info; + lnet_ping_info_t *info = rcd->rcd_pinginfo; + struct lnet_peer *gw = rcd->rcd_gateway; + lnet_route_t *rtr; - if (!avoid_asym_router_failure) - return -ENOENT; + if (!gw->lp_alive) + return; - if (rtr->lp_rcd == NULL) - return -EINVAL; + if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) + lnet_swap_pinginfo(info); - if (!rtr->lp_alive) - return -EINVAL; /* stale lp_rcd */ + /* NB always racing with network! */ + if (info->pi_magic != LNET_PROTO_PING_MAGIC) { + CDEBUG(D_NET, "%s: Unexpected magic %08x\n", + libcfs_nid2str(gw->lp_nid), info->pi_magic); + gw->lp_ping_version = LNET_PROTO_PING_UNKNOWN; + return; + } - info = rtr->lp_rcd->rcd_pinginfo; - LASSERT (info != NULL); + gw->lp_ping_version = info->pi_version; + if (gw->lp_ping_version == LNET_PROTO_PING_VERSION_1) + return; /* v1 doesn't carry NI status info */ - /* NB always racing with network! */ - if (info->pi_magic == __swab32(LNET_PROTO_PING_MAGIC)) { - lnet_swap_pinginfo(info); - } else if (info->pi_magic != LNET_PROTO_PING_MAGIC) { - CNETERR("%s: Unexpected magic %08x\n", - libcfs_nid2str(rtr->lp_nid), info->pi_magic); - return -EPROTO; - } + if (gw->lp_ping_version != LNET_PROTO_PING_VERSION) { + CDEBUG(D_NET, "%s: Unexpected version 0x%x\n", + libcfs_nid2str(gw->lp_nid), gw->lp_ping_version); + gw->lp_ping_version = LNET_PROTO_PING_UNKNOWN; + return; + } - if (info->pi_version == LNET_PROTO_PING_VERSION1) - return -ENOENT; /* v1 doesn't carry NI status info */ + cfs_list_for_each_entry(rtr, &gw->lp_routes, lr_gwlist) { + int ptl_status = LNET_NI_STATUS_INVALID; + int down = 0; + int up = 0; + int i; - if (info->pi_version != LNET_PROTO_PING_VERSION) { - CNETERR("%s: Unexpected version 0x%x\n", - libcfs_nid2str(rtr->lp_nid), info->pi_version); - return -EPROTO; - } + for (i = 0; i < info->pi_nnis && i < LNET_MAX_RTR_NIS; i++) { + lnet_ni_status_t *stat = &info->pi_ni[i]; + lnet_nid_t nid = stat->ns_nid; - for (i = 0; i < info->pi_nnis && i < LNET_MAX_RTR_NIS; i++) { - lnet_ni_status_t *stat = &info->pi_ni[i]; - lnet_nid_t nid = stat->ns_nid; + if (nid == LNET_NID_ANY) { + CDEBUG(D_NET, "%s: unexpected LNET_NID_ANY\n", + libcfs_nid2str(gw->lp_nid)); + gw->lp_ping_version = LNET_PROTO_PING_UNKNOWN; + return; + } - if (nid == LNET_NID_ANY) { - CNETERR("%s: unexpected LNET_NID_ANY\n", - libcfs_nid2str(rtr->lp_nid)); - return -EPROTO; - } + if (LNET_NETTYP(LNET_NIDNET(nid)) == LOLND) + continue; - if (LNET_NETTYP(LNET_NIDNET(nid)) == LOLND) - continue; + if (stat->ns_status == LNET_NI_STATUS_DOWN) { + if (LNET_NETTYP(LNET_NIDNET(nid)) != PTLLND) + down++; + else if (ptl_status != LNET_NI_STATUS_UP) + ptl_status = LNET_NI_STATUS_DOWN; + continue; + } + + if (stat->ns_status == LNET_NI_STATUS_UP) { + if (LNET_NIDNET(nid) == rtr->lr_net) { + up = 1; + break; + } + /* ptl NIs are considered down only when + * they're all down */ + if (LNET_NETTYP(LNET_NIDNET(nid)) == PTLLND) + ptl_status = LNET_NI_STATUS_UP; + continue; + } - if (stat->ns_status == LNET_NI_STATUS_DOWN) { - if (LNET_NETTYP(LNET_NIDNET(nid)) == PTLLND) - ptl_down = 1; - else - down++; - continue; - } + CDEBUG(D_NET, "%s: Unexpected status 0x%x\n", + libcfs_nid2str(gw->lp_nid), stat->ns_status); + gw->lp_ping_version = LNET_PROTO_PING_UNKNOWN; + return; + } - if (stat->ns_status != LNET_NI_STATUS_UP) { - CNETERR("%s: Unexpected status 0x%x\n", - libcfs_nid2str(rtr->lp_nid), stat->ns_status); - return -EPROTO; - } + if (up) { /* ignore downed NIs if NI for dest network is up */ + rtr->lr_downis = 0; + continue; + } + rtr->lr_downis = down + (ptl_status == LNET_NI_STATUS_DOWN); + } +} - /* ignore downed NIs if there's a NI up for dest network */ - if (LNET_NIDNET(nid) == net) - return 0; +static void +lnet_router_checker_event(lnet_event_t *event) +{ + /* CAVEAT EMPTOR: I'm called with lnet_res_locked */ + lnet_rc_data_t *rcd = event->md.user_ptr; + struct lnet_peer *lp; + + LASSERT(rcd != NULL); - if (LNET_NETTYP(LNET_NIDNET(nid)) == PTLLND) - ptl_up = 1; + if (event->unlinked) { + LNetInvalidateHandle(&rcd->rcd_mdh); + return; } - /* ptl NIs are considered down only when they're all down */ - return down + (ptl_up ? 0 : ptl_down); + LASSERT(event->type == LNET_EVENT_SEND || + event->type == LNET_EVENT_REPLY); + + lp = rcd->rcd_gateway; + LASSERT(lp != NULL); + + if (!lnet_isrouter(lp)) /* ignore if no longer a router */ + return; + + if (event->type == LNET_EVENT_SEND) { + lp->lp_ping_notsent = 0; /* NB: re-enable another ping */ + if (event->status == 0) + return; + } + + /* LNET_EVENT_REPLY */ + /* A successful REPLY means the router is up. If _any_ comms + * to the router fail I assume it's down (this will happen if + * we ping alive routers to try to detect router death before + * apps get burned). */ + + lnet_notify_locked(lp, 1, (event->status == 0), cfs_time_current()); + /* The router checker will wake up very shortly and do the + * actual notification. + * XXX If 'lp' stops being a router before then, it will still + * have the notification pending!!! */ + + if (avoid_asym_router_failure && event->status == 0) + lnet_parse_rc_info(rcd); } void @@ -654,65 +735,6 @@ lnet_wait_known_routerstate(void) } } -static void -lnet_router_checker_event (lnet_event_t *event) -{ - /* CAVEAT EMPTOR: I'm called with LNET_LOCKed and I'm not allowed to - * drop it (that's how come I see _every_ event, even ones that would - * overflow my EQ) */ - lnet_rc_data_t *rcd = event->md.user_ptr; - lnet_peer_t *lp; - lnet_nid_t nid; - - LASSERT(rcd != NULL); - - if (event->unlinked) { - LNetInvalidateHandle(&rcd->rcd_mdh); - return; - } - - LASSERT (event->type == LNET_EVENT_SEND || - event->type == LNET_EVENT_REPLY); - - nid = (event->type == LNET_EVENT_SEND) ? - event->target.nid : event->initiator.nid; - - lp = lnet_find_peer_locked(nid); - if (lp == NULL) { - /* router may have been removed */ - CDEBUG(D_NET, "Router %s not found\n", libcfs_nid2str(nid)); - return; - } - - if (event->type == LNET_EVENT_SEND) /* re-enable another ping */ - lp->lp_ping_notsent = 0; - - if (lnet_isrouter(lp) && /* ignore if no longer a router */ - (event->status != 0 || - event->type == LNET_EVENT_REPLY)) { - - /* A successful REPLY means the router is up. If _any_ comms - * to the router fail I assume it's down (this will happen if - * we ping alive routers to try to detect router death before - * apps get burned). */ - - lnet_notify_locked(lp, 1, (event->status == 0), - cfs_time_current()); - - /* The router checker will wake up very shortly and do the - * actual notification. - * XXX If 'lp' stops being a router before then, it will still - * have the notification pending!!! */ - } - - /* This decref will NOT drop LNET_LOCK (it had to have 1 ref when it - * was in the peer table and I've not dropped the lock, so no-one else - * can have reduced the refcount) */ - LASSERT(lp->lp_refcount > 1); - - lnet_peer_decref_locked(lp); -} - void lnet_update_ni_status(void) { @@ -757,6 +779,12 @@ lnet_destroy_rc_data (lnet_rc_data_t *rcd) /* detached from network */ LASSERT(LNetHandleIsInvalid(rcd->rcd_mdh)); + if (rcd->rcd_gateway != NULL) { + LNET_LOCK(); + lnet_peer_decref_locked(rcd->rcd_gateway); + LNET_UNLOCK(); + } + if (rcd->rcd_pinginfo != NULL) LIBCFS_FREE(rcd->rcd_pinginfo, LNET_PINGINFO_SIZE); @@ -813,6 +841,8 @@ lnet_create_rc_data_locked(lnet_peer_t *gateway) goto out; } + lnet_peer_addref_locked(gateway); + rcd->rcd_gateway = gateway; gateway->lp_rcd = rcd; return rcd; diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c index 258936f..2e8f0fa 100644 --- a/lnet/lnet/router_proc.c +++ b/lnet/lnet/router_proc.c @@ -331,8 +331,20 @@ int LL_PROC_PROTO(proc_lnet_routers) int pingsent = !peer->lp_ping_notsent; int last_ping = cfs_duration_sec(cfs_time_sub(now, peer->lp_ping_timestamp)); - int down_ni = lnet_router_down_ni(peer, - LNET_NIDNET(LNET_NID_ANY)); + int down_ni = 0; + lnet_route_t *rtr; + + if (peer->lp_ping_version == LNET_PROTO_PING_VERSION) { + cfs_list_for_each_entry(rtr, &peer->lp_routes, + lr_gwlist) { + /* downis on any route should be the + * number of downis on the gateway */ + if (rtr->lr_downis != 0) { + down_ni = rtr->lr_downis; + break; + } + } + } if (deadline == 0) s += snprintf(s, tmpstr + tmpsiz - s, -- 1.8.3.1