+ /* if no local_net found continue */
+ if (!local_net)
+ continue;
+
+ /*
+ * Iterate through the NIs in this local Net and select
+ * the NI to send from. The selection is determined by
+ * these 3 criterion in the following priority:
+ * 1. NUMA
+ * 2. NI available credits
+ * 3. Round Robin
+ */
+ best_ni = lnet_get_best_ni(local_net, best_ni, md_cpt);
+ }
+
+ if (!best_ni && !best_gw) {
+ lnet_net_unlock(cpt);
+ LCONSOLE_WARN("No local ni found to send from to %s\n",
+ libcfs_nid2str(dst_nid));
+ return -EINVAL;
+ }
+
+ if (!best_ni) {
+ best_ni = lnet_get_best_ni(best_gw->lpni_net, best_ni, md_cpt);
+ LASSERT(best_gw && best_ni);
+
+ /*
+ * We're going to route the message, so change the peer to
+ * the router.
+ */
+ LASSERT(best_gw->lpni_peer_net);
+ LASSERT(best_gw->lpni_peer_net->lpn_peer);
+ best_gw->lpni_gw_seq++;
+ peer = best_gw->lpni_peer_net->lpn_peer;
+ }
+
+ /*
+ * Now that we selected the NI to use increment its sequence
+ * number so the Round Robin algorithm will detect that it has
+ * been used and pick the next NI.
+ */
+ best_ni->ni_seq++;
+
+pick_peer:
+ /*
+ * At this point the best_ni is on a local network on which
+ * the peer has a peer_ni as well
+ */
+ peer_net = lnet_peer_get_net_locked(peer,
+ best_ni->ni_net->net_id);
+ /*
+ * peer_net is not available or the src_nid is explicitly defined
+ * and the peer_net for that src_nid is unhealthy. find a route to
+ * the destination nid.
+ */
+ if (!peer_net ||
+ (src_nid != LNET_NID_ANY &&
+ !lnet_is_peer_net_healthy_locked(peer_net))) {
+ best_gw = lnet_find_route_locked(best_ni->ni_net,
+ dst_nid,
+ rtr_nid);
+ /*
+ * if no route is found for that network then
+ * move onto the next peer_ni in the peer
+ */
+ if (!best_gw) {
+ lnet_net_unlock(cpt);
+ LCONSOLE_WARN("No route to peer from %s\n",
+ libcfs_nid2str(best_ni->ni_nid));
+ return -EHOSTUNREACH;
+ }
+
+ CDEBUG(D_NET, "Best route to %s via %s for %s %d\n",
+ libcfs_nid2str(dst_nid),
+ libcfs_nid2str(best_gw->lpni_nid),
+ lnet_msgtyp2str(msg->msg_type), msg->msg_len);
+
+ routing2 = true;
+ /*
+ * RULE: Each node considers only the next-hop
+ *
+ * We're going to route the message, so change the peer to
+ * the router.
+ */
+ LASSERT(best_gw->lpni_peer_net);
+ LASSERT(best_gw->lpni_peer_net->lpn_peer);
+ peer = best_gw->lpni_peer_net->lpn_peer;
+ } else if (!lnet_is_peer_net_healthy_locked(peer_net)) {
+ /*
+ * this peer_net is unhealthy but we still have an opportunity
+ * to find another peer_net that we can use
+ */
+ __u32 net_id = peer_net->lpn_net_id;
+ LCONSOLE_WARN("peer net %s unhealthy\n",
+ libcfs_net2str(net_id));
+ goto again;
+ }
+
+ /*
+ * Look at the peer NIs for the destination peer that connect
+ * to the chosen net. If a peer_ni is preferred when using the
+ * best_ni to communicate, we use that one. If there is no
+ * preferred peer_ni, or there are multiple preferred peer_ni,
+ * the available transmit credits are used. If the transmit
+ * credits are equal, we round-robin over the peer_ni.
+ */
+ lpni = NULL;
+ best_lpni_credits = INT_MIN;
+ preferred = false;
+ best_lpni = NULL;
+ while ((lpni = lnet_get_next_peer_ni_locked(peer, peer_net, lpni))) {
+ /*
+ * if this peer ni is not healthy just skip it, no point in
+ * examining it further
+ */
+ if (!lnet_is_peer_ni_healthy_locked(lpni))
+ continue;
+ ni_is_pref = lnet_peer_is_pref_nid_locked(lpni,
+ best_ni->ni_nid);
+
+ /* if this is a preferred peer use it */
+ if (!preferred && ni_is_pref) {
+ preferred = true;
+ } else if (preferred && !ni_is_pref) {
+ /*
+ * this is not the preferred peer so let's ignore
+ * it.
+ */
+ continue;
+ } else if (lpni->lpni_txcredits < best_lpni_credits) {
+ /*
+ * We already have a peer that has more credits
+ * available than this one. No need to consider
+ * this peer further.
+ */
+ continue;
+ } else if (lpni->lpni_txcredits == best_lpni_credits) {
+ /*
+ * The best peer found so far and the current peer
+ * have the same number of available credits let's
+ * make sure to select between them using Round
+ * Robin
+ */
+ if (best_lpni) {
+ if (best_lpni->lpni_seq <= lpni->lpni_seq)
+ continue;
+ }