From: Amir Shehata Date: Sat, 16 Feb 2019 01:59:40 +0000 (-0800) Subject: LU-9121 lnet: select best peer and local net X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=6224cfbb64624d18384ab57a70e6e6a0d2ef8da7;p=fs%2Flustre-release.git LU-9121 lnet: select best peer and local net Select the healthiest and highest priority peer and local net when sending a message. Lustre-change: https://review.whamcloud.com/34352 Lustre-commit: dff6587805ddad212ab48e5bedacbc7846542b7b Test-Parameters: trivial testlist=lnet-selftest,sanity-lnet Signed-off-by: Amir Shehata Change-Id: I42717e7fdc3226c6faa7c59c713f18422e27f2e5 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52444 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Frank Sehr --- diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index f681088..68793d2 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -575,6 +575,7 @@ int lnet_get_rtr_pool_cfg(int idx, struct lnet_ioctl_pool_cfg *pool_cfg); struct lnet_ni *lnet_get_next_ni_locked(struct lnet_net *mynet, struct lnet_ni *prev); struct lnet_ni *lnet_get_ni_idx_locked(int idx); +int lnet_get_net_healthv_locked(struct lnet_net *net); extern int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp, struct libcfs_ioctl_hdr __user *uparam); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index d30a2cf..fac6be2 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -390,6 +390,9 @@ struct lnet_net { * lnet/include/lnet/nidstr.h */ __u32 net_id; + /* round robin selection */ + __u32 net_seq; + /* total number of CPTs in the array */ __u32 net_ncpts; diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 0a48d34..4d4f9ae 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -3003,6 +3003,21 @@ lnet_get_ni_idx_locked(int idx) return NULL; } +int lnet_get_net_healthv_locked(struct lnet_net *net) +{ + struct lnet_ni *ni; + int best_healthv = 0; + int healthv; + + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + healthv = atomic_read(&ni->ni_healthv); + if (healthv > best_healthv) + best_healthv = healthv; + } + + return best_healthv; +} + struct lnet_ni * lnet_get_next_ni_locked(struct lnet_net *mynet, struct lnet_ni *prev) { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index fcfc9dd..cd88279 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1814,11 +1814,22 @@ lnet_handle_send(struct lnet_send_data *sd) __u32 routing = send_case & REMOTE_DST; struct lnet_rsp_tracker *rspt; - /* - * Increment sequence number of the selected peer so that we - * pick the next one in Round Robin. + /* Increment sequence number of the selected peer, peer net, + * local ni and local net so that we pick the next ones + * in Round Robin. */ best_lpni->lpni_seq++; + best_lpni->lpni_peer_net->lpn_seq++; + best_ni->ni_seq++; + best_ni->ni_net->net_seq++; + + CDEBUG(D_NET, "%s NI seq info: [%d:%d:%d] %s LPNI seq info [%d:%d:%d]\n", + libcfs_nid2str(best_ni->ni_nid), + best_ni->ni_seq, best_ni->ni_net->net_seq, + atomic_read(&best_ni->ni_tx_credits), + libcfs_nid2str(best_lpni->lpni_nid), + best_lpni->lpni_seq, best_lpni->lpni_peer_net->lpn_seq, + best_lpni->lpni_txcredits); /* * grab a reference on the peer_ni so it sticks around even if @@ -2013,8 +2024,7 @@ lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni, struct lnet_peer *peer, struct lnet_peer_net *peer_net, struct lnet_msg *msg, - int cpt, - bool incr_seq) + int cpt) { struct lnet_net *local_net; struct lnet_ni *best_ni; @@ -2034,9 +2044,6 @@ lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni, best_ni = lnet_get_best_ni(local_net, cur_best_ni, peer, peer_net, msg, cpt); - if (incr_seq && best_ni) - best_ni->ni_seq++; - return best_ni; } @@ -2273,8 +2280,7 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, lpn = gwni->lpni_peer_net; sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, gw, lpn, sd->sd_msg, - sd->sd_md_cpt, - true); + sd->sd_md_cpt); if (!sd->sd_best_ni) { CERROR("Internal Error. Expected local ni on %s but non found: %s\n", libcfs_net2str(lpn->lpn_net_id), @@ -2359,9 +2365,15 @@ struct lnet_ni * lnet_find_best_ni_on_local_net(struct lnet_peer *peer, struct lnet_msg *msg, int md_cpt, bool discovery) { - struct lnet_peer_net *peer_net = NULL; + struct lnet_peer_net *lpn = NULL; + struct lnet_peer_net *best_lpn = NULL; + struct lnet_net *net = NULL; + struct lnet_net *best_net = NULL; struct lnet_ni *best_ni = NULL; - int lpn_healthv = 0; + int best_lpn_healthv = 0; + int best_net_healthv = 0; + int net_healthv; + bool exit = false; /* * The peer can have multiple interfaces, some of them can be on @@ -2371,39 +2383,68 @@ lnet_find_best_ni_on_local_net(struct lnet_peer *peer, struct lnet_msg *msg, */ /* go through all the peer nets and find the best_ni */ - list_for_each_entry(peer_net, &peer->lp_peer_nets, lpn_peer_nets) { + list_for_each_entry(lpn, &peer->lp_peer_nets, lpn_peer_nets) { /* * The peer's list of nets can contain non-local nets. We * want to only examine the local ones. */ - if (!lnet_get_net_locked(peer_net->lpn_net_id)) + net = lnet_get_net_locked(lpn->lpn_net_id); + if (!net) continue; - /* always select the lpn with the best health */ - if (lpn_healthv <= peer_net->lpn_healthv) - lpn_healthv = peer_net->lpn_healthv; - else - continue; - - best_ni = lnet_find_best_ni_on_spec_net(best_ni, peer, peer_net, - msg, md_cpt, false); + net_healthv = lnet_get_net_healthv_locked(net); /* * if this is a discovery message and lp_disc_net_id is * specified then use that net to send the discovery on. */ - if (peer->lp_disc_net_id == peer_net->lpn_net_id && - discovery) - break; + if (peer->lp_disc_net_id == lpn->lpn_net_id && + discovery) { + exit = true; + goto select_lpn; + } + + if (!best_lpn) + goto select_lpn; + + /* always select the lpn with the best health */ + if (best_lpn_healthv > lpn->lpn_healthv) + continue; + else if (best_lpn_healthv < lpn->lpn_healthv) + goto select_lpn; + + /* select the preferred peer and local nets */ + if (best_net_healthv > net_healthv) + continue; + else if (best_net_healthv < net_healthv) + goto select_lpn; + + if (best_lpn->lpn_seq < lpn->lpn_seq) + continue; + else if (best_lpn->lpn_seq > lpn->lpn_seq) + goto select_lpn; - best_ni = lnet_find_best_ni_on_spec_net(best_ni, peer, - peer_net, msg, md_cpt, - false); + /* round robin over the local networks */ + if (best_net->net_seq <= net->net_seq) + continue; + +select_lpn: + best_net_healthv = net_healthv; + best_lpn_healthv = lpn->lpn_healthv; + best_lpn = lpn; + best_net = net; + + if (exit) + break; } - if (best_ni) - /* increment sequence number so we can round robin */ - best_ni->ni_seq++; + if (best_lpn) { + /* Select the best NI on the same net as best_lpn chosen + * above + */ + best_ni = lnet_find_best_ni_on_spec_net(NULL, peer, + best_lpn, msg, md_cpt); + } return best_ni; } @@ -2443,6 +2484,7 @@ static int lnet_select_preferred_best_ni(struct lnet_send_data *sd) { struct lnet_ni *best_ni = NULL; + struct lnet_peer_ni *best_lpni = sd->sd_best_lpni; /* * We must use a consistent source address when sending to a @@ -2462,19 +2504,20 @@ lnet_select_preferred_best_ni(struct lnet_send_data *sd) best_ni = lnet_find_existing_preferred_best_ni(sd->sd_best_lpni, sd->sd_cpt); - if (!best_ni) - best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, - sd->sd_best_lpni->lpni_peer_net, - sd->sd_msg, - sd->sd_md_cpt, - true); - /* If there is no best_ni we don't have a route */ if (!best_ni) { - CERROR("no path to %s from net %s\n", - libcfs_nid2str(sd->sd_best_lpni->lpni_nid), - libcfs_net2str(sd->sd_best_lpni->lpni_net->net_id)); - return -EHOSTUNREACH; + best_ni = + lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, + sd->sd_best_lpni->lpni_peer_net, + sd->sd_msg, + sd->sd_md_cpt); + /* If there is no best_ni we don't have a route */ + if (!best_ni) { + CERROR("no path to %s from net %s\n", + libcfs_nid2str(best_lpni->lpni_nid), + libcfs_net2str(best_lpni->lpni_net->net_id)); + return -EHOSTUNREACH; + } } sd->sd_best_ni = best_ni; @@ -2524,8 +2567,7 @@ lnet_handle_any_local_nmr_dst(struct lnet_send_data *sd) sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, sd->sd_msg, - sd->sd_md_cpt, - true); + sd->sd_md_cpt); if (!sd->sd_best_ni) { CERROR("Unable to forward message to %s. No local NI available\n", libcfs_nid2str(sd->sd_dst_nid)); @@ -2558,8 +2600,7 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd) sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, - sd->sd_msg, sd->sd_md_cpt, - true); + sd->sd_msg, sd->sd_md_cpt); if (!sd->sd_best_ni) { /*