From ca8958189198d8272551948dba7b36dea77c3180 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Wed, 7 Aug 2019 20:33:13 -0500 Subject: [PATCH] LU-12646 lnet: Prefer route specified by rtr_nid Restore an optimization that was initially added under LU-11413. For routed REPLY and ACK we should preferably use the same router from which the GET/PUT was receieved. Cray-bug-id: LUS-8008 Test-Parameters: trivial Signed-off-by: Chris Horn Change-Id: Ia3059cddf70e1c477d90acdd90c13b4c5a292f4f Reviewed-on: https://review.whamcloud.com/35737 Tested-by: jenkins Reviewed-by: Alexandr Boyko Tested-by: Maloo Reviewed-by: Shaun Tancheff Reviewed-by: Neil Brown Reviewed-by: Oleg Drokin --- lnet/lnet/lib-move.c | 130 +++++++++++++++++++++++++++++---------------------- lnet/lnet/lib-msg.c | 2 - 2 files changed, 75 insertions(+), 57 deletions(-) diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 262e432..3d9bed4 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1516,7 +1516,7 @@ lnet_compare_routes(struct lnet_route *r1, struct lnet_route *r2, static struct lnet_route * lnet_find_route_locked(struct lnet_net *net, __u32 remote_net, - lnet_nid_t rtr_nid, struct lnet_route **prev_route, + struct lnet_route **prev_route, struct lnet_peer_ni **gwni) { struct lnet_peer_ni *best_gw_ni = NULL; @@ -1528,9 +1528,6 @@ lnet_find_route_locked(struct lnet_net *net, __u32 remote_net, struct lnet_peer *lp; int rc; - /* If @rtr_nid is not LNET_NID_ANY, return the gateway with - * rtr_nid nid, otherwise find the best gateway I can use */ - rnet = lnet_find_rnet_locked(remote_net); if (rnet == NULL) return NULL; @@ -1847,13 +1844,14 @@ lnet_handle_send(struct lnet_send_data *sd) rc = lnet_post_send_locked(msg, 0); if (!rc) - CDEBUG(D_NET, "TRACE: %s(%s:%s) -> %s(%s:%s) : %s try# %d\n", + CDEBUG(D_NET, "TRACE: %s(%s:%s) -> %s(%s:%s) %s : %s try# %d\n", libcfs_nid2str(msg->msg_hdr.src_nid), libcfs_nid2str(msg->msg_txni->ni_nid), libcfs_nid2str(sd->sd_src_nid), libcfs_nid2str(msg->msg_hdr.dest_nid), libcfs_nid2str(sd->sd_dst_nid), libcfs_nid2str(msg->msg_txpeer->lpni_nid), + libcfs_nid2str(sd->sd_rtr_nid), lnet_msgtyp2str(msg->msg_type), msg->msg_retry_count); return rc; @@ -2028,69 +2026,90 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, struct lnet_peer **gw_peer) { int rc; + __u32 local_lnet; struct lnet_peer *gw; struct lnet_peer *lp; struct lnet_peer_net *lpn; struct lnet_peer_net *best_lpn = NULL; struct lnet_remotenet *rnet; - struct lnet_route *best_route; - struct lnet_route *last_route; + struct lnet_route *best_route = NULL; + struct lnet_route *last_route = NULL; struct lnet_peer_ni *lpni = NULL; struct lnet_peer_ni *gwni = NULL; lnet_nid_t src_nid = sd->sd_src_nid; - /* we've already looked up the initial lpni using dst_nid */ - lpni = sd->sd_best_lpni; - /* the peer tree must be in existence */ - LASSERT(lpni && lpni->lpni_peer_net && lpni->lpni_peer_net->lpn_peer); - lp = lpni->lpni_peer_net->lpn_peer; + /* If a router nid was specified then we are replying to a GET or + * sending an ACK. In this case we use the gateway associated with the + * specified router nid. + */ + if (sd->sd_rtr_nid != LNET_NID_ANY) { + gwni = lnet_find_peer_ni_locked(sd->sd_rtr_nid); + if (!gwni) { + CERROR("No peer NI for gateway %s\n", + libcfs_nid2str(sd->sd_rtr_nid)); + return -EHOSTUNREACH; + } + gw = gwni->lpni_peer_net->lpn_peer; + lnet_peer_ni_decref_locked(gwni); + local_lnet = LNET_NIDNET(sd->sd_rtr_nid); + } else { + /* we've already looked up the initial lpni using dst_nid */ + lpni = sd->sd_best_lpni; + /* the peer tree must be in existence */ + LASSERT(lpni && lpni->lpni_peer_net && + lpni->lpni_peer_net->lpn_peer); + lp = lpni->lpni_peer_net->lpn_peer; + + list_for_each_entry(lpn, &lp->lp_peer_nets, lpn_peer_nets) { + /* is this remote network reachable? */ + rnet = lnet_find_rnet_locked(lpn->lpn_net_id); + if (!rnet) + continue; - list_for_each_entry(lpn, &lp->lp_peer_nets, lpn_peer_nets) { - /* is this remote network reachable? */ - rnet = lnet_find_rnet_locked(lpn->lpn_net_id); - if (!rnet) - continue; + if (!best_lpn) + best_lpn = lpn; + + if (best_lpn->lpn_seq <= lpn->lpn_seq) + continue; - if (!best_lpn) best_lpn = lpn; + } - if (best_lpn->lpn_seq <= lpn->lpn_seq) - continue; + if (!best_lpn) { + CERROR("peer %s has no available nets\n", + libcfs_nid2str(sd->sd_dst_nid)); + return -EHOSTUNREACH; + } - best_lpn = lpn; - } + sd->sd_best_lpni = lnet_find_best_lpni_on_net(sd, lp, best_lpn->lpn_net_id); + if (!sd->sd_best_lpni) { + CERROR("peer %s down\n", + libcfs_nid2str(sd->sd_dst_nid)); + return -EHOSTUNREACH; + } - if (!best_lpn) { - CERROR("peer %s has no available nets \n", - libcfs_nid2str(sd->sd_dst_nid)); - return -EHOSTUNREACH; - } + best_route = lnet_find_route_locked(NULL, best_lpn->lpn_net_id, + &last_route, &gwni); + if (!best_route) { + CERROR("no route to %s from %s\n", + libcfs_nid2str(dst_nid), + libcfs_nid2str(src_nid)); + return -EHOSTUNREACH; + } - sd->sd_best_lpni = lnet_find_best_lpni_on_net(sd, lp, best_lpn->lpn_net_id); - if (!sd->sd_best_lpni) { - CERROR("peer %s down\n", libcfs_nid2str(sd->sd_dst_nid)); - return -EHOSTUNREACH; - } + if (!gwni) { + CERROR("Internal Error. Route expected to %s from %s\n", + libcfs_nid2str(dst_nid), + libcfs_nid2str(src_nid)); + return -EFAULT; + } - best_route = lnet_find_route_locked(NULL, best_lpn->lpn_net_id, - sd->sd_rtr_nid, &last_route, - &gwni); - if (!best_route) { - CERROR("no route to %s from %s\n", - libcfs_nid2str(dst_nid), libcfs_nid2str(src_nid)); - return -EHOSTUNREACH; - } + gw = best_route->lr_gateway; + LASSERT(gw == gwni->lpni_peer_net->lpn_peer); + local_lnet = best_route->lr_lnet; - if (!gwni) { - CERROR("Internal Error. Route expected to %s from %s\n", - libcfs_nid2str(dst_nid), - libcfs_nid2str(src_nid)); - return -EFAULT; } - gw = best_route->lr_gateway; - LASSERT(gw == gwni->lpni_peer_net->lpn_peer); - /* * Discover this gateway if it hasn't already been discovered. * This means we might delay the message until discovery has @@ -2105,14 +2124,13 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, if (!sd->sd_best_ni) sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, gw, lnet_peer_get_net_locked(gw, - best_route->lr_lnet), + local_lnet), sd->sd_md_cpt, true); if (!sd->sd_best_ni) { - CERROR("Internal Error. Expected local ni on %s " - "but non found :%s\n", - libcfs_net2str(best_route->lr_lnet), + CERROR("Internal Error. Expected local ni on %s but non found :%s\n", + libcfs_net2str(local_lnet), libcfs_nid2str(sd->sd_src_nid)); return -EFAULT; } @@ -2124,9 +2142,11 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, * increment the sequence numbers since now we're sure we're * going to use this path */ - LASSERT(best_route && last_route); - best_route->lr_seq = last_route->lr_seq + 1; - best_lpn->lpn_seq++; + if (sd->sd_rtr_nid == LNET_NID_ANY) { + LASSERT(best_route && last_route); + best_route->lr_seq = last_route->lr_seq + 1; + best_lpn->lpn_seq++; + } return 0; } diff --git a/lnet/lnet/lib-msg.c b/lnet/lnet/lib-msg.c index fcf021a..84bf23d 100644 --- a/lnet/lnet/lib-msg.c +++ b/lnet/lnet/lib-msg.c @@ -391,8 +391,6 @@ lnet_complete_msg_locked(struct lnet_msg *msg, int cpt) msg->msg_hdr.msg.ack.match_bits = msg->msg_ev.match_bits; msg->msg_hdr.msg.ack.mlength = cpu_to_le32(msg->msg_ev.mlength); - /* NB: we probably want to use NID of msg::msg_from as 3rd - * parameter (router NID) if it's routed message */ rc = lnet_send(msg->msg_ev.target.nid, msg, msg->msg_from); lnet_net_lock(cpt); -- 1.8.3.1