X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Flib-move.c;h=cdd84d6d47cbf8f40da222f7c80420cf8f30c4de;hp=9420d6a0da1fd1b3b2547bbf679c3db56fa2e342;hb=7e89b556ea7dc4b4cf5e6bce3b1da6bed0190e8b;hpb=79ab0535622782c82636cee47918dc4b19983144 diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 9420d6a..cdd84d6 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -39,6 +39,7 @@ #include #include +#include #include static int local_nid_dist_zero = 1; @@ -189,13 +190,15 @@ void lnet_usr_translate_stats(struct lnet_ioctl_element_msg_stats *msg_stats, } int -lnet_fail_nid(lnet_nid_t nid, unsigned int threshold) +lnet_fail_nid(lnet_nid_t nid4, unsigned int threshold) { struct lnet_test_peer *tp; struct list_head *el; struct list_head *next; + struct lnet_nid nid; LIST_HEAD(cull); + lnet_nid4_to_nid(nid4, &nid); /* NB: use lnet_net_lock(0) to serialize operations on test peers */ if (threshold != 0) { /* Adding a new entry */ @@ -218,8 +221,8 @@ lnet_fail_nid(lnet_nid_t nid, unsigned int threshold) tp = list_entry(el, struct lnet_test_peer, tp_list); if (tp->tp_threshold == 0 || /* needs culling anyway */ - nid == LNET_NID_ANY || /* removing all entries */ - tp->tp_nid == nid) { /* matched this one */ + LNET_NID_IS_ANY(&nid) || /* removing all entries */ + nid_same(&tp->tp_nid, &nid)) { /* matched this one */ list_move(&tp->tp_list, &cull); } } @@ -236,14 +239,16 @@ lnet_fail_nid(lnet_nid_t nid, unsigned int threshold) } static int -fail_peer (lnet_nid_t nid, int outgoing) +fail_peer(lnet_nid_t nid4, int outgoing) { struct lnet_test_peer *tp; struct list_head *el; struct list_head *next; + struct lnet_nid nid; LIST_HEAD(cull); int fail = 0; + lnet_nid4_to_nid(nid4, &nid); /* NB: use lnet_net_lock(0) to serialize operations on test peers */ lnet_net_lock(0); @@ -261,8 +266,8 @@ fail_peer (lnet_nid_t nid, int outgoing) continue; } - if (tp->tp_nid == LNET_NID_ANY || /* fail every peer */ - nid == tp->tp_nid) { /* fail this peer */ + if (LNET_NID_IS_ANY(&tp->tp_nid) || /* fail every peer */ + nid_same(&nid, &tp->tp_nid)) { /* fail this peer */ fail = 1; if (tp->tp_threshold != LNET_MD_THRESH_INF) { @@ -725,14 +730,14 @@ lnet_prep_send(struct lnet_msg *msg, int type, struct lnet_process_id target, msg->msg_hdr.payload_length = cpu_to_le32(len); } -static void +void lnet_ni_send(struct lnet_ni *ni, struct lnet_msg *msg) { void *priv = msg->msg_private; int rc; LASSERT(!in_interrupt()); - LASSERT(ni->ni_nid == LNET_NID_LO_0 || + LASSERT(nid_is_lo0(&ni->ni_nid) || (msg->msg_txcredit && msg->msg_peertxcredit)); rc = (ni->ni_net->net_lnd->lnd_send)(ni, priv, msg); @@ -758,7 +763,7 @@ lnet_ni_eager_recv(struct lnet_ni *ni, struct lnet_msg *msg) if (rc != 0) { CERROR("recv from %s / send to %s aborted: " "eager_recv failed %d\n", - libcfs_nid2str(msg->msg_rxpeer->lpni_nid), + libcfs_nidstr(&msg->msg_rxpeer->lpni_nid), libcfs_id2str(msg->msg_target), rc); LASSERT(rc < 0); /* required by my callers */ } @@ -842,7 +847,7 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send) /* can't get here if we're sending to the loopback interface */ if (the_lnet.ln_loni) - LASSERT(lp->lpni_nid != the_lnet.ln_loni->ni_nid); + LASSERT(!nid_same(&lp->lpni_nid, &the_lnet.ln_loni->ni_nid)); /* NB 'lp' is always the next hop */ if ((msg->msg_target.pid & LNET_PID_USERFLAG) == 0 && @@ -927,6 +932,12 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send) } } + if (unlikely(!list_empty(&the_lnet.ln_delay_rules)) && + lnet_delay_rule_match_locked(&msg->msg_hdr, msg)) { + msg->msg_tx_delayed = 1; + return LNET_CREDIT_WAIT; + } + /* unset the tx_delay flag as we're going to send it now */ msg->msg_tx_delayed = 0; @@ -1315,10 +1326,11 @@ lnet_select_peer_ni(struct lnet_ni *best_ni, lnet_nid_t dst_nid, * preferred, then let's use it */ if (best_ni) { - lpni_is_preferred = lnet_peer_is_pref_nid_locked(lpni, - best_ni->ni_nid); + /* FIXME need to handle large-addr nid */ + lpni_is_preferred = lnet_peer_is_pref_nid_locked( + lpni, lnet_nid_to_nid4(&best_ni->ni_nid)); CDEBUG(D_NET, "%s lpni_is_preferred = %d\n", - libcfs_nid2str(best_ni->ni_nid), + libcfs_nidstr(&best_ni->ni_nid), lpni_is_preferred); } else { lpni_is_preferred = false; @@ -1329,8 +1341,8 @@ lnet_select_peer_ni(struct lnet_ni *best_ni, lnet_nid_t dst_nid, if (best_lpni) CDEBUG(D_NET, "n:[%s, %s] h:[%d, %d] p:[%d, %d] c:[%d, %d] s:[%d, %d]\n", - libcfs_nid2str(lpni->lpni_nid), - libcfs_nid2str(best_lpni->lpni_nid), + libcfs_nidstr(&lpni->lpni_nid), + libcfs_nidstr(&best_lpni->lpni_nid), lpni_healthv, best_lpni_healthv, lpni_sel_prio, best_sel_prio, lpni->lpni_txcredits, best_lpni_credits, @@ -1399,7 +1411,7 @@ select_lpni: } CDEBUG(D_NET, "sd_best_lpni = %s\n", - libcfs_nid2str(best_lpni->lpni_nid)); + libcfs_nidstr(&best_lpni->lpni_nid)); return best_lpni; } @@ -1503,7 +1515,7 @@ lnet_find_route_locked(struct lnet_remotenet *rnet, __u32 src_net, list_for_each_entry(route, &rnet->lrn_routes, lr_list) { if (!lnet_is_route_alive(route)) continue; - gw_pnid = route->lr_gateway->lp_primary_nid; + gw_pnid = lnet_nid_to_nid4(&route->lr_gateway->lp_primary_nid); /* no protection on below fields, but it's harmless */ if (last_route && (last_route->lr_seq - route->lr_seq < 0)) @@ -1599,16 +1611,38 @@ lnet_find_route_locked(struct lnet_remotenet *rnet, __u32 src_net, return best_route; } +static inline unsigned int +lnet_dev_prio_of_md(struct lnet_ni *ni, unsigned int dev_idx) +{ + if (dev_idx == UINT_MAX) + return UINT_MAX; + + if (!ni || !ni->ni_net || !ni->ni_net->net_lnd || + !ni->ni_net->net_lnd->lnd_get_dev_prio) + return UINT_MAX; + + return ni->ni_net->net_lnd->lnd_get_dev_prio(ni, dev_idx); +} + static struct lnet_ni * lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, struct lnet_peer *peer, struct lnet_peer_net *peer_net, - int md_cpt) + struct lnet_msg *msg, int md_cpt) { - struct lnet_ni *ni = NULL; + struct lnet_libmd *md = msg->msg_md; + unsigned int offset = msg->msg_offset; unsigned int shortest_distance; + struct lnet_ni *ni = NULL; int best_credits; int best_healthv; __u32 best_sel_prio; + unsigned int best_dev_prio; + unsigned int dev_idx = UINT_MAX; + struct page *page = lnet_get_first_page(md, offset); + msg->msg_rdma_force = lnet_is_rdma_only_page(page); + + if (msg->msg_rdma_force) + dev_idx = lnet_get_dev_idx(page); /* * If there is no peer_ni that we can send to on this network, @@ -1620,9 +1654,11 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, if (best_ni == NULL) { best_sel_prio = LNET_MAX_SELECTION_PRIORITY; shortest_distance = UINT_MAX; + best_dev_prio = UINT_MAX; best_credits = INT_MIN; best_healthv = 0; } else { + best_dev_prio = lnet_dev_prio_of_md(best_ni, dev_idx); shortest_distance = cfs_cpt_distance(lnet_cpt_table(), md_cpt, best_ni->ni_dev_cpt); best_credits = atomic_read(&best_ni->ni_tx_credits); @@ -1636,6 +1672,7 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, int ni_healthv; int ni_fatal; __u32 ni_sel_prio; + unsigned int ni_dev_prio; ni_credits = atomic_read(&ni->ni_tx_credits); ni_healthv = atomic_read(&ni->ni_healthv); @@ -1651,6 +1688,8 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, md_cpt, ni->ni_dev_cpt); + ni_dev_prio = lnet_dev_prio_of_md(ni, dev_idx); + /* * All distances smaller than the NUMA range * are treated equally. @@ -1659,20 +1698,20 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, distance = lnet_numa_range; /* - * Select on health, shorter distance, available - * credits, then round-robin. + * Select on health, selection policy, direct dma prio, + * shorter distance, available credits, then round-robin. */ if (ni_fatal) continue; if (best_ni) - CDEBUG(D_NET, "compare ni %s [c:%d, d:%d, s:%d, p:%u] with best_ni %s [c:%d, d:%d, s:%d, p:%u]\n", - libcfs_nid2str(ni->ni_nid), ni_credits, distance, - ni->ni_seq, ni_sel_prio, - (best_ni) ? libcfs_nid2str(best_ni->ni_nid) + CDEBUG(D_NET, "compare ni %s [c:%d, d:%d, s:%d, p:%u, g:%u] with best_ni %s [c:%d, d:%d, s:%d, p:%u, g:%u]\n", + libcfs_nidstr(&ni->ni_nid), ni_credits, distance, + ni->ni_seq, ni_sel_prio, ni_dev_prio, + (best_ni) ? libcfs_nidstr(&best_ni->ni_nid) : "not selected", best_credits, shortest_distance, (best_ni) ? best_ni->ni_seq : 0, - best_sel_prio); + best_sel_prio, best_dev_prio); else goto select_ni; @@ -1686,6 +1725,11 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, else if (ni_sel_prio < best_sel_prio) goto select_ni; + if (ni_dev_prio > best_dev_prio) + continue; + else if (ni_dev_prio < best_dev_prio) + goto select_ni; + if (distance > shortest_distance) continue; else if (distance < shortest_distance) @@ -1701,6 +1745,7 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, select_ni: best_sel_prio = ni_sel_prio; + best_dev_prio = ni_dev_prio; shortest_distance = distance; best_healthv = ni_healthv; best_ni = ni; @@ -1708,7 +1753,7 @@ select_ni: } CDEBUG(D_NET, "selected best_ni %s\n", - (best_ni) ? libcfs_nid2str(best_ni->ni_nid) : "no selection"); + (best_ni) ? libcfs_nidstr(&best_ni->ni_nid) : "no selection"); return best_ni; } @@ -1767,11 +1812,12 @@ lnet_handle_lo_send(struct lnet_send_data *sd) /* No send credit hassles with LOLND */ lnet_ni_addref_locked(the_lnet.ln_loni, cpt); - msg->msg_hdr.dest_nid = cpu_to_le64(the_lnet.ln_loni->ni_nid); + msg->msg_hdr.dest_nid = + cpu_to_le64(lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid)); if (!msg->msg_routing) msg->msg_hdr.src_nid = - cpu_to_le64(the_lnet.ln_loni->ni_nid); - msg->msg_target.nid = the_lnet.ln_loni->ni_nid; + cpu_to_le64(lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid)); + msg->msg_target.nid = lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid); lnet_msg_commit(msg, cpt); msg->msg_txni = the_lnet.ln_loni; @@ -1795,17 +1841,17 @@ lnet_handle_send(struct lnet_send_data *sd) * local ni and local net so that we pick the next ones * in Round Robin. */ - best_lpni->lpni_seq++; best_lpni->lpni_peer_net->lpn_seq++; - best_ni->ni_seq++; + best_lpni->lpni_seq = best_lpni->lpni_peer_net->lpn_seq; best_ni->ni_net->net_seq++; + best_ni->ni_seq = best_ni->ni_net->net_seq; CDEBUG(D_NET, "%s NI seq info: [%d:%d:%d:%u] %s LPNI seq info [%d:%d:%d:%u]\n", - libcfs_nid2str(best_ni->ni_nid), + libcfs_nidstr(&best_ni->ni_nid), best_ni->ni_seq, best_ni->ni_net->net_seq, atomic_read(&best_ni->ni_tx_credits), best_ni->ni_sel_priority, - libcfs_nid2str(best_lpni->lpni_nid), + libcfs_nidstr(&best_lpni->lpni_nid), best_lpni->lpni_seq, best_lpni->lpni_peer_net->lpn_seq, best_lpni->lpni_txcredits, best_lpni->lpni_sel_priority); @@ -1825,7 +1871,7 @@ lnet_handle_send(struct lnet_send_data *sd) * the configuration has changed. We don't have a hold on the best_ni * yet, and it may have vanished. */ - cpt2 = lnet_cpt_of_nid_locked(best_lpni->lpni_nid, best_ni); + cpt2 = lnet_cpt_of_nid_locked(&best_lpni->lpni_nid, best_ni); if (sd->sd_cpt != cpt2) { __u32 seq = lnet_get_dlc_seq_locked(); lnet_net_unlock(sd->sd_cpt); @@ -1856,7 +1902,8 @@ lnet_handle_send(struct lnet_send_data *sd) * what was originally set in the target or it will be the NID of * a router if this message should be routed */ - msg->msg_target.nid = msg->msg_txpeer->lpni_nid; + /* FIXME handle large-addr nids */ + msg->msg_target.nid = lnet_nid_to_nid4(&msg->msg_txpeer->lpni_nid); /* * lnet_msg_commit assigns the correct cpt to the message, which @@ -1871,7 +1918,8 @@ lnet_handle_send(struct lnet_send_data *sd) * originator and set it here. */ if (!msg->msg_routing) - msg->msg_hdr.src_nid = cpu_to_le64(msg->msg_txni->ni_nid); + msg->msg_hdr.src_nid = + cpu_to_le64(lnet_nid_to_nid4(&msg->msg_txni->ni_nid)); if (routing) { msg->msg_target_is_router = 1; @@ -1886,13 +1934,16 @@ lnet_handle_send(struct lnet_send_data *sd) * lnet_select_pathway() function and is never changed. * It's safe to use it here. */ - msg->msg_hdr.dest_nid = cpu_to_le64(final_dst_lpni->lpni_nid); + /* FIXME handle large-addr nid */ + msg->msg_hdr.dest_nid = + cpu_to_le64(lnet_nid_to_nid4(&final_dst_lpni->lpni_nid)); } else { /* * if we're not routing set the dest_nid to the best peer * ni NID that we picked earlier in the algorithm. */ - msg->msg_hdr.dest_nid = cpu_to_le64(msg->msg_txpeer->lpni_nid); + msg->msg_hdr.dest_nid = + cpu_to_le64(lnet_nid_to_nid4(&msg->msg_txpeer->lpni_nid)); } /* @@ -1902,9 +1953,10 @@ lnet_handle_send(struct lnet_send_data *sd) if (msg->msg_md) { rspt = msg->msg_md->md_rspt_ptr; if (rspt) { - rspt->rspt_next_hop_nid = msg->msg_txpeer->lpni_nid; + rspt->rspt_next_hop_nid = + msg->msg_txpeer->lpni_nid; CDEBUG(D_NET, "rspt_next_hop_nid = %s\n", - libcfs_nid2str(rspt->rspt_next_hop_nid)); + libcfs_nidstr(&rspt->rspt_next_hop_nid)); } } @@ -1913,11 +1965,11 @@ lnet_handle_send(struct lnet_send_data *sd) if (!rc) CDEBUG(D_NET, "TRACE: %s(%s:%s) -> %s(%s:%s) %s : %s try# %d\n", libcfs_nid2str(msg->msg_hdr.src_nid), - libcfs_nid2str(msg->msg_txni->ni_nid), + libcfs_nidstr(&msg->msg_txni->ni_nid), libcfs_nid2str(sd->sd_src_nid), libcfs_nid2str(msg->msg_hdr.dest_nid), libcfs_nid2str(sd->sd_dst_nid), - libcfs_nid2str(msg->msg_txpeer->lpni_nid), + libcfs_nidstr(&msg->msg_txpeer->lpni_nid), libcfs_nid2str(sd->sd_rtr_nid), lnet_msgtyp2str(msg->msg_type), msg->msg_retry_count); @@ -1931,9 +1983,10 @@ lnet_set_non_mr_pref_nid(struct lnet_peer_ni *lpni, struct lnet_ni *lni, if (!lnet_peer_is_multi_rail(lpni->lpni_peer_net->lpn_peer) && !lnet_msg_is_response(msg) && lpni->lpni_pref_nnids == 0) { CDEBUG(D_NET, "Setting preferred local NID %s on NMR peer %s\n", - libcfs_nid2str(lni->ni_nid), - libcfs_nid2str(lpni->lpni_nid)); - lnet_peer_ni_set_non_mr_pref_nid(lpni, lni->ni_nid); + libcfs_nidstr(&lni->ni_nid), + libcfs_nidstr(&lpni->lpni_nid)); + lnet_peer_ni_set_non_mr_pref_nid( + lpni, lnet_nid_to_nid4(&lni->ni_nid)); } } @@ -1986,7 +2039,8 @@ lnet_handle_spec_local_mr_dst(struct lnet_send_data *sd) } if (sd->sd_best_lpni && - sd->sd_best_lpni->lpni_nid == the_lnet.ln_loni->ni_nid) + nid_same(&sd->sd_best_lpni->lpni_nid, + &the_lnet.ln_loni->ni_nid)) return lnet_handle_lo_send(sd); else if (sd->sd_best_lpni) return lnet_handle_send(sd); @@ -2002,6 +2056,7 @@ struct lnet_ni * lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni, struct lnet_peer *peer, struct lnet_peer_net *peer_net, + struct lnet_msg *msg, int cpt) { struct lnet_net *local_net; @@ -2020,7 +2075,7 @@ lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni, * 3. Round Robin */ best_ni = lnet_get_best_ni(local_net, cur_best_ni, - peer, peer_net, cpt); + peer, peer_net, msg, cpt); return best_ni; } @@ -2053,7 +2108,7 @@ lnet_initiate_peer_discovery(struct lnet_peer_ni *lpni, struct lnet_msg *msg, return rc; } - new_lpni = lnet_find_peer_ni_locked(lpni->lpni_nid); + new_lpni = lnet_find_peer_ni_locked(lnet_nid_to_nid4(&lpni->lpni_nid)); if (!new_lpni) { lnet_peer_ni_decref_locked(lpni); return -ENOENT; @@ -2086,7 +2141,7 @@ lnet_initiate_peer_discovery(struct lnet_peer_ni *lpni, struct lnet_msg *msg, lnet_peer_ni_decref_locked(new_lpni); CDEBUG(D_NET, "msg %p delayed. %s pending discovery\n", - msg, libcfs_nid2str(peer->lp_primary_nid)); + msg, libcfs_nidstr(&peer->lp_primary_nid)); return LNET_DC_WAIT; } @@ -2098,7 +2153,6 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, struct lnet_peer **gw_peer) { int rc; - __u32 local_lnet; struct lnet_peer *gw; struct lnet_peer *lp; struct lnet_peer_net *lpn; @@ -2110,8 +2164,9 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, struct lnet_peer_ni *gwni = NULL; bool route_found = false; lnet_nid_t src_nid = (sd->sd_src_nid != LNET_NID_ANY) ? sd->sd_src_nid : - (sd->sd_best_ni != NULL) ? sd->sd_best_ni->ni_nid : - LNET_NID_ANY; + (sd->sd_best_ni != NULL) + ? lnet_nid_to_nid4(&sd->sd_best_ni->ni_nid) + : LNET_NID_ANY; int best_lpn_healthv = 0; __u32 best_lpn_sel_prio = LNET_MAX_SELECTION_PRIORITY; @@ -2127,10 +2182,8 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd, if (gwni) { gw = gwni->lpni_peer_net->lpn_peer; lnet_peer_ni_decref_locked(gwni); - if (gw->lp_rtr_refcount) { - local_lnet = LNET_NIDNET(sd->sd_rtr_nid); + if (gw->lp_rtr_refcount) route_found = true; - } } else { CWARN("No peer NI for gateway %s. Attempting to find an alternative route.\n", libcfs_nid2str(sd->sd_rtr_nid)); @@ -2246,7 +2299,6 @@ use_lpn: gw = best_route->lr_gateway; LASSERT(gw == gwni->lpni_peer_net->lpn_peer); - local_lnet = best_route->lr_lnet; } /* @@ -2254,22 +2306,21 @@ use_lpn: * This means we might delay the message until discovery has * completed */ - sd->sd_msg->msg_src_nid_param = sd->sd_src_nid; rc = lnet_initiate_peer_discovery(gwni, sd->sd_msg, sd->sd_cpt); if (rc) return rc; - if (!sd->sd_best_ni) - sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, gw, - lnet_peer_get_net_locked(gw, - local_lnet), - sd->sd_md_cpt); - if (!sd->sd_best_ni) { - CERROR("Internal Error. Expected local ni on %s but non found :%s\n", - libcfs_net2str(local_lnet), - libcfs_nid2str(sd->sd_src_nid)); - return -EFAULT; + lpn = gwni->lpni_peer_net; + sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, gw, lpn, + sd->sd_msg, + sd->sd_md_cpt); + if (!sd->sd_best_ni) { + CERROR("Internal Error. Expected local ni on %s but non found: %s\n", + libcfs_net2str(lpn->lpn_net_id), + libcfs_nid2str(sd->sd_src_nid)); + return -EFAULT; + } } *gw_lpni = gwni; @@ -2346,7 +2397,7 @@ lnet_handle_spec_router_dst(struct lnet_send_data *sd) struct lnet_ni * lnet_find_best_ni_on_local_net(struct lnet_peer *peer, int md_cpt, - bool discovery) + struct lnet_msg *msg, bool discovery) { struct lnet_peer_net *lpn = NULL; struct lnet_peer_net *best_lpn = NULL; @@ -2443,8 +2494,8 @@ select_lpn: /* Select the best NI on the same net as best_lpn chosen * above */ - best_ni = lnet_find_best_ni_on_spec_net(NULL, peer, - best_lpn, md_cpt); + best_ni = lnet_find_best_ni_on_spec_net(NULL, peer, best_lpn, + msg, md_cpt); } return best_ni; @@ -2506,11 +2557,12 @@ lnet_select_preferred_best_ni(struct lnet_send_data *sd) best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, + sd->sd_msg, sd->sd_md_cpt); /* If there is no best_ni we don't have a route */ if (!best_ni) { CERROR("no path to %s from net %s\n", - libcfs_nid2str(best_lpni->lpni_nid), + libcfs_nidstr(&best_lpni->lpni_nid), libcfs_net2str(best_lpni->lpni_net->net_id)); return -EHOSTUNREACH; } @@ -2562,6 +2614,7 @@ lnet_handle_any_local_nmr_dst(struct lnet_send_data *sd) sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, + sd->sd_msg, sd->sd_md_cpt); if (!sd->sd_best_ni) { CERROR("Unable to forward message to %s. No local NI available\n", @@ -2595,6 +2648,7 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd) sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, + sd->sd_msg, sd->sd_md_cpt); if (!sd->sd_best_ni) { @@ -2619,6 +2673,7 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd) */ sd->sd_best_ni = lnet_find_best_ni_on_local_net(sd->sd_peer, sd->sd_md_cpt, + sd->sd_msg, lnet_msg_discovery(sd->sd_msg)); if (sd->sd_best_ni) { sd->sd_best_lpni = @@ -2633,7 +2688,8 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd) * network */ if (sd->sd_best_lpni && - sd->sd_best_lpni->lpni_nid == the_lnet.ln_loni->ni_nid) { + nid_same(&sd->sd_best_lpni->lpni_nid, + &the_lnet.ln_loni->ni_nid)) { /* * in case we initially started with a routed * destination, let's reset to local @@ -3168,7 +3224,7 @@ lnet_finalize_expired_responses(void) if (ktime_compare(now, rspt->rspt_deadline) >= 0 || the_lnet.ln_mt_state == LNET_MT_STATE_SHUTDOWN) { struct lnet_peer_ni *lpni; - lnet_nid_t nid; + struct lnet_nid nid; md = lnet_handle2md(&rspt->rspt_mdh); if (!md) { @@ -3226,7 +3282,7 @@ lnet_finalize_expired_responses(void) CDEBUG(D_NET, "Response timeout: md = %p: nid = %s\n", - md, libcfs_nid2str(nid)); + md, libcfs_nidstr(&nid)); /* * If there is a timeout on the response @@ -3234,7 +3290,7 @@ lnet_finalize_expired_responses(void) * value so that we don't use it */ lnet_net_lock(0); - lpni = lnet_find_peer_ni_locked(nid); + lpni = lnet_peer_ni_find_locked(&nid); if (lpni) { lnet_handle_remote_failure_locked(lpni); lnet_peer_ni_decref_locked(lpni); @@ -3408,7 +3464,7 @@ lnet_recover_local_nis(void) lnet_net_unlock(0); CDEBUG(D_NET, "attempting to recover local ni: %s\n", - libcfs_nid2str(ni->ni_nid)); + libcfs_nidstr(&ni->ni_nid)); lnet_ni_lock(ni); if (!(ni->ni_recovery_state & LNET_NI_RECOVERY_PENDING)) { @@ -3418,7 +3474,7 @@ lnet_recover_local_nis(void) LIBCFS_ALLOC(ev_info, sizeof(*ev_info)); if (!ev_info) { CERROR("out of memory. Can't recover %s\n", - libcfs_nid2str(ni->ni_nid)); + libcfs_nidstr(&ni->ni_nid)); lnet_ni_lock(ni); ni->ni_recovery_state &= ~LNET_NI_RECOVERY_PENDING; @@ -3432,7 +3488,8 @@ lnet_recover_local_nis(void) * We'll unlink the mdh in this case below. */ LNetInvalidateMDHandle(&ni->ni_ping_mdh); - nid = ni->ni_nid; + /* FIXME need to handle large-addr nid */ + nid = lnet_nid_to_nid4(&ni->ni_nid); /* * remove the NI from the local queue and drop the @@ -3673,7 +3730,7 @@ lnet_recover_peer_nis(void) LIBCFS_ALLOC(ev_info, sizeof(*ev_info)); if (!ev_info) { CERROR("out of memory. Can't recover %s\n", - libcfs_nid2str(lpni->lpni_nid)); + libcfs_nidstr(&lpni->lpni_nid)); spin_lock(&lpni->lpni_lock); lpni->lpni_state &= ~LNET_PEER_NI_RECOVERY_PENDING; spin_unlock(&lpni->lpni_lock); @@ -3683,7 +3740,8 @@ lnet_recover_peer_nis(void) /* look at the comments in lnet_recover_local_nis() */ mdh = lpni->lpni_recovery_ping_mdh; LNetInvalidateMDHandle(&lpni->lpni_recovery_ping_mdh); - nid = lpni->lpni_nid; + /* FIXME handle large-addr nid */ + nid = lnet_nid_to_nid4(&lpni->lpni_nid); lnet_net_lock(0); list_del_init(&lpni->lpni_recovery); lnet_peer_ni_decref_locked(lpni); @@ -4210,11 +4268,12 @@ lnet_parse_get(struct lnet_ni *ni, struct lnet_msg *msg, int rdma_get) lnet_ni_recv(ni, msg->msg_private, NULL, 0, 0, 0, 0); msg->msg_receiving = 0; - rc = lnet_send(ni->ni_nid, msg, msg->msg_from); + /* FIXME need to handle large-addr nid */ + rc = lnet_send(lnet_nid_to_nid4(&ni->ni_nid), msg, msg->msg_from); if (rc < 0) { /* didn't get as far as lnet_ni_send() */ CERROR("%s: Unable to send REPLY for GET from %s: %d\n", - libcfs_nid2str(ni->ni_nid), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(info.mi_id), rc); lnet_finalize(msg, rc); @@ -4245,7 +4304,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg) if (md == NULL || md->md_threshold == 0 || md->md_me != NULL) { CNETERR("%s: Dropping REPLY from %s for %s " "MD %#llx.%#llx\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src), (md == NULL) ? "invalid" : "inactive", hdr->msg.reply.dst_wmd.wh_interface_cookie, hdr->msg.reply.dst_wmd.wh_object_cookie); @@ -4266,7 +4325,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg) (md->md_options & LNET_MD_TRUNCATE) == 0) { CNETERR("%s: Dropping REPLY from %s length %d " "for MD %#llx would overflow (%d)\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src), rlength, hdr->msg.reply.dst_wmd.wh_object_cookie, mlength); lnet_res_unlock(cpt); @@ -4274,7 +4333,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg) } CDEBUG(D_NET, "%s: Reply from %s of length %d/%d into md %#llx\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src), mlength, rlength, hdr->msg.reply.dst_wmd.wh_object_cookie); lnet_msg_attach_md(msg, md, 0, mlength); @@ -4314,7 +4373,7 @@ lnet_parse_ack(struct lnet_ni *ni, struct lnet_msg *msg) /* Don't moan; this is expected */ CDEBUG(D_NET, "%s: Dropping ACK from %s to %s MD %#llx.%#llx\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src), (md == NULL) ? "invalid" : "inactive", hdr->msg.ack.dst_wmd.wh_interface_cookie, hdr->msg.ack.dst_wmd.wh_object_cookie); @@ -4327,7 +4386,7 @@ lnet_parse_ack(struct lnet_ni *ni, struct lnet_msg *msg) } CDEBUG(D_NET, "%s: ACK from %s into md %#llx\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src), hdr->msg.ack.dst_wmd.wh_object_cookie); lnet_msg_attach_md(msg, md, 0, 0); @@ -4439,12 +4498,13 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, dest_pid = le32_to_cpu(hdr->dest_pid); payload_length = le32_to_cpu(hdr->payload_length); - for_me = (ni->ni_nid == dest_nid); + /* FIXME handle large-addr nids */ + for_me = (lnet_nid_to_nid4(&ni->ni_nid) == dest_nid); cpt = lnet_cpt_of_nid(from_nid, ni); CDEBUG(D_NET, "TRACE: %s(%s) <- %s : %s - %s\n", libcfs_nid2str(dest_nid), - libcfs_nid2str(ni->ni_nid), + libcfs_nidstr(&ni->ni_nid), libcfs_nid2str(src_nid), lnet_msgtyp2str(type), (for_me) ? "for me" : "routed"); @@ -4501,7 +4561,7 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, * or malicious so we chop them off at the knees :) */ if (!for_me) { - if (LNET_NIDNET(dest_nid) == LNET_NIDNET(ni->ni_nid)) { + if (LNET_NIDNET(dest_nid) == LNET_NID_NET(&ni->ni_nid)) { /* should have gone direct */ CERROR("%s, src %s: Bad dest nid %s " "(should have been sent direct)\n", @@ -4511,7 +4571,7 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, return -EPROTO; } - if (lnet_islocalnid(dest_nid)) { + if (lnet_islocalnid4(dest_nid)) { /* dest is another local NI; sender should have used * this node's NID on its own network */ CERROR("%s, src %s: Bad dest nid %s " @@ -4552,8 +4612,9 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, goto drop; } + /* FIXME need to support large-addr nid */ if (!list_empty(&the_lnet.ln_drop_rules) && - lnet_drop_rule_match(hdr, ni->ni_nid, NULL)) { + lnet_drop_rule_match(hdr, lnet_nid_to_nid4(&ni->ni_nid), NULL)) { CDEBUG(D_NET, "%s, src %s, dst %s: Dropping %s to simulate silent message loss\n", libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), @@ -4597,13 +4658,15 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, } lnet_net_lock(cpt); - lpni = lnet_nid2peerni_locked(from_nid, ni->ni_nid, cpt); + /* FIXME support large-addr nid */ + lpni = lnet_nid2peerni_locked(from_nid, lnet_nid_to_nid4(&ni->ni_nid), + cpt); if (IS_ERR(lpni)) { lnet_net_unlock(cpt); - CERROR("%s, src %s: Dropping %s " - "(error %ld looking up sender)\n", + rc = PTR_ERR(lpni); + CERROR("%s, src %s: Dropping %s (error %d looking up sender)\n", libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), - lnet_msgtyp2str(type), PTR_ERR(lpni)); + lnet_msgtyp2str(type), rc); lnet_msg_free(msg); if (rc == -ESHUTDOWN) /* We are shutting down. Don't do anything more */ @@ -4644,6 +4707,8 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid, } } if (lnet_drop_asym_route && for_me && !found) { + /* Drop ref taken by lnet_nid2peerni_locked() */ + lnet_peer_ni_decref_locked(lpni); lnet_net_unlock(cpt); /* we would not use from_nid to route a message to * src_nid @@ -5022,7 +5087,7 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg) if (msg == NULL) { CERROR("%s: Dropping REPLY from %s: can't allocate msg\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id)); + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id)); goto drop; } @@ -5033,7 +5098,7 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg) if (getmd->md_threshold == 0) { CERROR("%s: Dropping REPLY from %s for inactive MD %p\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id), + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id), getmd); lnet_res_unlock(cpt); goto drop; @@ -5042,10 +5107,11 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg) LASSERT(getmd->md_offset == 0); CDEBUG(D_NET, "%s: Reply from %s md %p\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id), getmd); + libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id), getmd); /* setup information for lnet_build_msg_event */ - msg->msg_initiator = getmsg->msg_txpeer->lpni_peer_net->lpn_peer->lp_primary_nid; + msg->msg_initiator = + lnet_nid_to_nid4(&getmsg->msg_txpeer->lpni_peer_net->lpn_peer->lp_primary_nid); msg->msg_from = peer_id.nid; msg->msg_type = LNET_MSG_GET; /* flag this msg as an "optimized" GET */ msg->msg_hdr.src_nid = peer_id.nid; @@ -5242,6 +5308,7 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) int cpt; __u32 order = 2; struct list_head *rn_list; + bool matched_dstnet = false; /* if !local_nid_dist_zero, I don't return a distance of 0 ever * (when lustre sees a distance of 0, it substitutes 0@lo), so I @@ -5253,7 +5320,8 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) cpt = lnet_net_lock_current(); while ((ni = lnet_get_next_ni_locked(NULL, ni))) { - if (ni->ni_nid == dstnid) { + /* FIXME support large-addr nid */ + if (lnet_nid_to_nid4(&ni->ni_nid) == dstnid) { if (srcnidp != NULL) *srcnidp = dstnid; if (orderp != NULL) { @@ -5267,25 +5335,41 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) return local_nid_dist_zero ? 0 : 1; } - if (LNET_NIDNET(ni->ni_nid) == dstnet) { - /* Check if ni was originally created in - * current net namespace. - * If not, assign order above 0xffff0000, - * to make this ni not a priority. */ - if (current->nsproxy && - !net_eq(ni->ni_net_ns, current->nsproxy->net_ns)) - order += 0xffff0000; - if (srcnidp != NULL) - *srcnidp = ni->ni_nid; - if (orderp != NULL) - *orderp = order; - lnet_net_unlock(cpt); - return 1; + if (!matched_dstnet && LNET_NID_NET(&ni->ni_nid) == dstnet) { + matched_dstnet = true; + /* We matched the destination net, but we may have + * additional local NIs to inspect. + * + * We record the nid and order as appropriate, but + * they may be overwritten if we match local NI above. + */ + if (srcnidp) + /* FIXME support large-addr nids */ + *srcnidp = lnet_nid_to_nid4(&ni->ni_nid); + + if (orderp) { + /* Check if ni was originally created in + * current net namespace. + * If not, assign order above 0xffff0000, + * to make this ni not a priority. + */ + if (current->nsproxy && + !net_eq(ni->ni_net_ns, + current->nsproxy->net_ns)) + *orderp = order + 0xffff0000; + else + *orderp = order; + } } order++; } + if (matched_dstnet) { + lnet_net_unlock(cpt); + return 1; + } + rn_list = lnet_net2rnethash(dstnet); list_for_each(e, rn_list) { rnet = list_entry(e, struct lnet_remotenet, lrn_list); @@ -5317,7 +5401,8 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) net = lnet_get_net_locked(shortest->lr_lnet); LASSERT(net); ni = lnet_get_next_ni_locked(net, NULL); - *srcnidp = ni->ni_nid; + /* FIXME support large-addr nids */ + *srcnidp = lnet_nid_to_nid4(&ni->ni_nid); } if (orderp != NULL) *orderp = order;