Whamcloud - gitweb
LU-10391 lnet: extend rspt_next_hop_nid in lnet_rsp_tracker
[fs/lustre-release.git] / lnet / lnet / lib-move.c
index e458bbf..67ab256 100644 (file)
@@ -39,6 +39,7 @@
 
 #include <lnet/lib-lnet.h>
 #include <linux/nsproxy.h>
+#include <lnet/lnet_rdma.h>
 #include <net/net_namespace.h>
 
 static int local_nid_dist_zero = 1;
@@ -732,7 +733,7 @@ lnet_ni_send(struct lnet_ni *ni, struct lnet_msg *msg)
        int rc;
 
        LASSERT(!in_interrupt());
-       LASSERT(ni->ni_nid == LNET_NID_LO_0 ||
+       LASSERT(nid_is_lo0(&ni->ni_nid) ||
                (msg->msg_txcredit && msg->msg_peertxcredit));
 
        rc = (ni->ni_net->net_lnd->lnd_send)(ni, priv, msg);
@@ -758,7 +759,7 @@ lnet_ni_eager_recv(struct lnet_ni *ni, struct lnet_msg *msg)
        if (rc != 0) {
                CERROR("recv from %s / send to %s aborted: "
                       "eager_recv failed %d\n",
-                      libcfs_nid2str(msg->msg_rxpeer->lpni_nid),
+                      libcfs_nidstr(&msg->msg_rxpeer->lpni_nid),
                       libcfs_id2str(msg->msg_target), rc);
                LASSERT(rc < 0); /* required by my callers */
        }
@@ -842,7 +843,7 @@ lnet_post_send_locked(struct lnet_msg *msg, int do_send)
 
        /* can't get here if we're sending to the loopback interface */
        if (the_lnet.ln_loni)
-               LASSERT(lp->lpni_nid != the_lnet.ln_loni->ni_nid);
+               LASSERT(!nid_same(&lp->lpni_nid, &the_lnet.ln_loni->ni_nid));
 
        /* NB 'lp' is always the next hop */
        if ((msg->msg_target.pid & LNET_PID_USERFLAG) == 0 &&
@@ -1321,10 +1322,11 @@ lnet_select_peer_ni(struct lnet_ni *best_ni, lnet_nid_t dst_nid,
                 * preferred, then let's use it
                 */
                if (best_ni) {
-                       lpni_is_preferred = lnet_peer_is_pref_nid_locked(lpni,
-                                                               best_ni->ni_nid);
+                       /* FIXME need to handle large-addr nid */
+                       lpni_is_preferred = lnet_peer_is_pref_nid_locked(
+                               lpni, lnet_nid_to_nid4(&best_ni->ni_nid));
                        CDEBUG(D_NET, "%s lpni_is_preferred = %d\n",
-                              libcfs_nid2str(best_ni->ni_nid),
+                              libcfs_nidstr(&best_ni->ni_nid),
                               lpni_is_preferred);
                } else {
                        lpni_is_preferred = false;
@@ -1335,8 +1337,8 @@ lnet_select_peer_ni(struct lnet_ni *best_ni, lnet_nid_t dst_nid,
 
                if (best_lpni)
                        CDEBUG(D_NET, "n:[%s, %s] h:[%d, %d] p:[%d, %d] c:[%d, %d] s:[%d, %d]\n",
-                               libcfs_nid2str(lpni->lpni_nid),
-                               libcfs_nid2str(best_lpni->lpni_nid),
+                               libcfs_nidstr(&lpni->lpni_nid),
+                               libcfs_nidstr(&best_lpni->lpni_nid),
                                lpni_healthv, best_lpni_healthv,
                                lpni_sel_prio, best_sel_prio,
                                lpni->lpni_txcredits, best_lpni_credits,
@@ -1405,7 +1407,7 @@ select_lpni:
        }
 
        CDEBUG(D_NET, "sd_best_lpni = %s\n",
-              libcfs_nid2str(best_lpni->lpni_nid));
+              libcfs_nidstr(&best_lpni->lpni_nid));
 
        return best_lpni;
 }
@@ -1509,7 +1511,7 @@ lnet_find_route_locked(struct lnet_remotenet *rnet, __u32 src_net,
        list_for_each_entry(route, &rnet->lrn_routes, lr_list) {
                if (!lnet_is_route_alive(route))
                        continue;
-               gw_pnid = route->lr_gateway->lp_primary_nid;
+               gw_pnid = lnet_nid_to_nid4(&route->lr_gateway->lp_primary_nid);
 
                /* no protection on below fields, but it's harmless */
                if (last_route && (last_route->lr_seq - route->lr_seq < 0))
@@ -1605,16 +1607,38 @@ lnet_find_route_locked(struct lnet_remotenet *rnet, __u32 src_net,
        return best_route;
 }
 
+static inline unsigned int
+lnet_dev_prio_of_md(struct lnet_ni *ni, unsigned int dev_idx)
+{
+       if (dev_idx == UINT_MAX)
+               return UINT_MAX;
+
+       if (!ni || !ni->ni_net || !ni->ni_net->net_lnd ||
+           !ni->ni_net->net_lnd->lnd_get_dev_prio)
+               return UINT_MAX;
+
+       return ni->ni_net->net_lnd->lnd_get_dev_prio(ni, dev_idx);
+}
+
 static struct lnet_ni *
 lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
                 struct lnet_peer *peer, struct lnet_peer_net *peer_net,
-                int md_cpt)
+                struct lnet_msg *msg, int md_cpt)
 {
-       struct lnet_ni *ni = NULL;
+       struct lnet_libmd *md = msg->msg_md;
+       unsigned int offset = msg->msg_offset;
        unsigned int shortest_distance;
+       struct lnet_ni *ni = NULL;
        int best_credits;
        int best_healthv;
        __u32 best_sel_prio;
+       unsigned int best_dev_prio;
+       unsigned int dev_idx = UINT_MAX;
+       struct page *page = lnet_get_first_page(md, offset);
+       msg->msg_rdma_force = lnet_is_rdma_only_page(page);
+
+       if (msg->msg_rdma_force)
+               dev_idx = lnet_get_dev_idx(page);
 
        /*
         * If there is no peer_ni that we can send to on this network,
@@ -1626,9 +1650,11 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
        if (best_ni == NULL) {
                best_sel_prio = LNET_MAX_SELECTION_PRIORITY;
                shortest_distance = UINT_MAX;
+               best_dev_prio = UINT_MAX;
                best_credits = INT_MIN;
                best_healthv = 0;
        } else {
+               best_dev_prio = lnet_dev_prio_of_md(best_ni, dev_idx);
                shortest_distance = cfs_cpt_distance(lnet_cpt_table(), md_cpt,
                                                     best_ni->ni_dev_cpt);
                best_credits = atomic_read(&best_ni->ni_tx_credits);
@@ -1642,6 +1668,7 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
                int ni_healthv;
                int ni_fatal;
                __u32 ni_sel_prio;
+               unsigned int ni_dev_prio;
 
                ni_credits = atomic_read(&ni->ni_tx_credits);
                ni_healthv = atomic_read(&ni->ni_healthv);
@@ -1657,6 +1684,8 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
                                            md_cpt,
                                            ni->ni_dev_cpt);
 
+               ni_dev_prio = lnet_dev_prio_of_md(ni, dev_idx);
+
                /*
                 * All distances smaller than the NUMA range
                 * are treated equally.
@@ -1665,20 +1694,20 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
                        distance = lnet_numa_range;
 
                /*
-                * Select on health, shorter distance, available
-                * credits, then round-robin.
+                * Select on health, selection policy, direct dma prio,
+                * shorter distance, available credits, then round-robin.
                 */
                if (ni_fatal)
                        continue;
 
                if (best_ni)
-                       CDEBUG(D_NET, "compare ni %s [c:%d, d:%d, s:%d, p:%u] with best_ni %s [c:%d, d:%d, s:%d, p:%u]\n",
-                              libcfs_nid2str(ni->ni_nid), ni_credits, distance,
-                              ni->ni_seq, ni_sel_prio,
-                              (best_ni) ? libcfs_nid2str(best_ni->ni_nid)
+                       CDEBUG(D_NET, "compare ni %s [c:%d, d:%d, s:%d, p:%u, g:%u] with best_ni %s [c:%d, d:%d, s:%d, p:%u, g:%u]\n",
+                              libcfs_nidstr(&ni->ni_nid), ni_credits, distance,
+                              ni->ni_seq, ni_sel_prio, ni_dev_prio,
+                              (best_ni) ? libcfs_nidstr(&best_ni->ni_nid)
                               : "not selected", best_credits, shortest_distance,
                               (best_ni) ? best_ni->ni_seq : 0,
-                              best_sel_prio);
+                              best_sel_prio, best_dev_prio);
                else
                        goto select_ni;
 
@@ -1692,6 +1721,11 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
                else if (ni_sel_prio < best_sel_prio)
                        goto select_ni;
 
+               if (ni_dev_prio > best_dev_prio)
+                       continue;
+               else if (ni_dev_prio < best_dev_prio)
+                       goto select_ni;
+
                if (distance > shortest_distance)
                        continue;
                else if (distance < shortest_distance)
@@ -1707,6 +1741,7 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni,
 
 select_ni:
                best_sel_prio = ni_sel_prio;
+               best_dev_prio = ni_dev_prio;
                shortest_distance = distance;
                best_healthv = ni_healthv;
                best_ni = ni;
@@ -1714,7 +1749,7 @@ select_ni:
        }
 
        CDEBUG(D_NET, "selected best_ni %s\n",
-              (best_ni) ? libcfs_nid2str(best_ni->ni_nid) : "no selection");
+              (best_ni) ? libcfs_nidstr(&best_ni->ni_nid) : "no selection");
 
        return best_ni;
 }
@@ -1773,11 +1808,12 @@ lnet_handle_lo_send(struct lnet_send_data *sd)
 
        /* No send credit hassles with LOLND */
        lnet_ni_addref_locked(the_lnet.ln_loni, cpt);
-       msg->msg_hdr.dest_nid = cpu_to_le64(the_lnet.ln_loni->ni_nid);
+       msg->msg_hdr.dest_nid =
+               cpu_to_le64(lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid));
        if (!msg->msg_routing)
                msg->msg_hdr.src_nid =
-                       cpu_to_le64(the_lnet.ln_loni->ni_nid);
-       msg->msg_target.nid = the_lnet.ln_loni->ni_nid;
+                       cpu_to_le64(lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid));
+       msg->msg_target.nid = lnet_nid_to_nid4(&the_lnet.ln_loni->ni_nid);
        lnet_msg_commit(msg, cpt);
        msg->msg_txni = the_lnet.ln_loni;
 
@@ -1807,11 +1843,11 @@ lnet_handle_send(struct lnet_send_data *sd)
        best_ni->ni_net->net_seq++;
 
        CDEBUG(D_NET, "%s NI seq info: [%d:%d:%d:%u] %s LPNI seq info [%d:%d:%d:%u]\n",
-              libcfs_nid2str(best_ni->ni_nid),
+              libcfs_nidstr(&best_ni->ni_nid),
               best_ni->ni_seq, best_ni->ni_net->net_seq,
               atomic_read(&best_ni->ni_tx_credits),
               best_ni->ni_sel_priority,
-              libcfs_nid2str(best_lpni->lpni_nid),
+              libcfs_nidstr(&best_lpni->lpni_nid),
               best_lpni->lpni_seq, best_lpni->lpni_peer_net->lpn_seq,
               best_lpni->lpni_txcredits,
               best_lpni->lpni_sel_priority);
@@ -1831,7 +1867,7 @@ lnet_handle_send(struct lnet_send_data *sd)
         * the configuration has changed. We don't have a hold on the best_ni
         * yet, and it may have vanished.
         */
-       cpt2 = lnet_cpt_of_nid_locked(best_lpni->lpni_nid, best_ni);
+       cpt2 = lnet_cpt_of_nid_locked(&best_lpni->lpni_nid, best_ni);
        if (sd->sd_cpt != cpt2) {
                __u32 seq = lnet_get_dlc_seq_locked();
                lnet_net_unlock(sd->sd_cpt);
@@ -1862,7 +1898,8 @@ lnet_handle_send(struct lnet_send_data *sd)
         * what was originally set in the target or it will be the NID of
         * a router if this message should be routed
         */
-       msg->msg_target.nid = msg->msg_txpeer->lpni_nid;
+       /* FIXME handle large-addr nids */
+       msg->msg_target.nid = lnet_nid_to_nid4(&msg->msg_txpeer->lpni_nid);
 
        /*
         * lnet_msg_commit assigns the correct cpt to the message, which
@@ -1877,7 +1914,8 @@ lnet_handle_send(struct lnet_send_data *sd)
         * originator and set it here.
         */
        if (!msg->msg_routing)
-               msg->msg_hdr.src_nid = cpu_to_le64(msg->msg_txni->ni_nid);
+               msg->msg_hdr.src_nid =
+                       cpu_to_le64(lnet_nid_to_nid4(&msg->msg_txni->ni_nid));
 
        if (routing) {
                msg->msg_target_is_router = 1;
@@ -1892,13 +1930,16 @@ lnet_handle_send(struct lnet_send_data *sd)
                 * lnet_select_pathway() function and is never changed.
                 * It's safe to use it here.
                 */
-               msg->msg_hdr.dest_nid = cpu_to_le64(final_dst_lpni->lpni_nid);
+               /* FIXME handle large-addr nid */
+               msg->msg_hdr.dest_nid =
+                       cpu_to_le64(lnet_nid_to_nid4(&final_dst_lpni->lpni_nid));
        } else {
                /*
                 * if we're not routing set the dest_nid to the best peer
                 * ni NID that we picked earlier in the algorithm.
                 */
-               msg->msg_hdr.dest_nid = cpu_to_le64(msg->msg_txpeer->lpni_nid);
+               msg->msg_hdr.dest_nid =
+                       cpu_to_le64(lnet_nid_to_nid4(&msg->msg_txpeer->lpni_nid));
        }
 
        /*
@@ -1908,9 +1949,10 @@ lnet_handle_send(struct lnet_send_data *sd)
        if (msg->msg_md) {
                rspt = msg->msg_md->md_rspt_ptr;
                if (rspt) {
-                       rspt->rspt_next_hop_nid = msg->msg_txpeer->lpni_nid;
+                       rspt->rspt_next_hop_nid =
+                               msg->msg_txpeer->lpni_nid;
                        CDEBUG(D_NET, "rspt_next_hop_nid = %s\n",
-                              libcfs_nid2str(rspt->rspt_next_hop_nid));
+                              libcfs_nidstr(&rspt->rspt_next_hop_nid));
                }
        }
 
@@ -1919,11 +1961,11 @@ lnet_handle_send(struct lnet_send_data *sd)
        if (!rc)
                CDEBUG(D_NET, "TRACE: %s(%s:%s) -> %s(%s:%s) %s : %s try# %d\n",
                       libcfs_nid2str(msg->msg_hdr.src_nid),
-                      libcfs_nid2str(msg->msg_txni->ni_nid),
+                      libcfs_nidstr(&msg->msg_txni->ni_nid),
                       libcfs_nid2str(sd->sd_src_nid),
                       libcfs_nid2str(msg->msg_hdr.dest_nid),
                       libcfs_nid2str(sd->sd_dst_nid),
-                      libcfs_nid2str(msg->msg_txpeer->lpni_nid),
+                      libcfs_nidstr(&msg->msg_txpeer->lpni_nid),
                       libcfs_nid2str(sd->sd_rtr_nid),
                       lnet_msgtyp2str(msg->msg_type), msg->msg_retry_count);
 
@@ -1937,9 +1979,10 @@ lnet_set_non_mr_pref_nid(struct lnet_peer_ni *lpni, struct lnet_ni *lni,
        if (!lnet_peer_is_multi_rail(lpni->lpni_peer_net->lpn_peer) &&
            !lnet_msg_is_response(msg) && lpni->lpni_pref_nnids == 0) {
                CDEBUG(D_NET, "Setting preferred local NID %s on NMR peer %s\n",
-                      libcfs_nid2str(lni->ni_nid),
-                      libcfs_nid2str(lpni->lpni_nid));
-               lnet_peer_ni_set_non_mr_pref_nid(lpni, lni->ni_nid);
+                      libcfs_nidstr(&lni->ni_nid),
+                      libcfs_nidstr(&lpni->lpni_nid));
+               lnet_peer_ni_set_non_mr_pref_nid(
+                       lpni, lnet_nid_to_nid4(&lni->ni_nid));
        }
 }
 
@@ -1992,7 +2035,8 @@ lnet_handle_spec_local_mr_dst(struct lnet_send_data *sd)
        }
 
        if (sd->sd_best_lpni &&
-           sd->sd_best_lpni->lpni_nid == the_lnet.ln_loni->ni_nid)
+           nid_same(&sd->sd_best_lpni->lpni_nid,
+                     &the_lnet.ln_loni->ni_nid))
                return lnet_handle_lo_send(sd);
        else if (sd->sd_best_lpni)
                return lnet_handle_send(sd);
@@ -2008,6 +2052,7 @@ struct lnet_ni *
 lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni,
                              struct lnet_peer *peer,
                              struct lnet_peer_net *peer_net,
+                             struct lnet_msg *msg,
                              int cpt)
 {
        struct lnet_net *local_net;
@@ -2026,7 +2071,7 @@ lnet_find_best_ni_on_spec_net(struct lnet_ni *cur_best_ni,
         *      3. Round Robin
         */
        best_ni = lnet_get_best_ni(local_net, cur_best_ni,
-                                  peer, peer_net, cpt);
+                                  peer, peer_net, msg, cpt);
 
        return best_ni;
 }
@@ -2059,7 +2104,7 @@ lnet_initiate_peer_discovery(struct lnet_peer_ni *lpni, struct lnet_msg *msg,
                return rc;
        }
 
-       new_lpni = lnet_find_peer_ni_locked(lpni->lpni_nid);
+       new_lpni = lnet_find_peer_ni_locked(lnet_nid_to_nid4(&lpni->lpni_nid));
        if (!new_lpni) {
                lnet_peer_ni_decref_locked(lpni);
                return -ENOENT;
@@ -2092,7 +2137,7 @@ lnet_initiate_peer_discovery(struct lnet_peer_ni *lpni, struct lnet_msg *msg,
        lnet_peer_ni_decref_locked(new_lpni);
 
        CDEBUG(D_NET, "msg %p delayed. %s pending discovery\n",
-              msg, libcfs_nid2str(peer->lp_primary_nid));
+              msg, libcfs_nidstr(&peer->lp_primary_nid));
 
        return LNET_DC_WAIT;
 }
@@ -2115,8 +2160,9 @@ lnet_handle_find_routed_path(struct lnet_send_data *sd,
        struct lnet_peer_ni *gwni = NULL;
        bool route_found = false;
        lnet_nid_t src_nid = (sd->sd_src_nid != LNET_NID_ANY) ? sd->sd_src_nid :
-               (sd->sd_best_ni != NULL) ? sd->sd_best_ni->ni_nid :
-               LNET_NID_ANY;
+               (sd->sd_best_ni != NULL)
+               ? lnet_nid_to_nid4(&sd->sd_best_ni->ni_nid)
+               : LNET_NID_ANY;
        int best_lpn_healthv = 0;
        __u32 best_lpn_sel_prio = LNET_MAX_SELECTION_PRIORITY;
 
@@ -2263,6 +2309,7 @@ use_lpn:
        if (!sd->sd_best_ni) {
                lpn = gwni->lpni_peer_net;
                sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL, gw, lpn,
+                                                              sd->sd_msg,
                                                               sd->sd_md_cpt);
                if (!sd->sd_best_ni) {
                        CERROR("Internal Error. Expected local ni on %s but non found: %s\n",
@@ -2346,7 +2393,7 @@ lnet_handle_spec_router_dst(struct lnet_send_data *sd)
 
 struct lnet_ni *
 lnet_find_best_ni_on_local_net(struct lnet_peer *peer, int md_cpt,
-                              bool discovery)
+                              struct lnet_msg *msg, bool discovery)
 {
        struct lnet_peer_net *lpn = NULL;
        struct lnet_peer_net *best_lpn = NULL;
@@ -2443,8 +2490,8 @@ select_lpn:
                /* Select the best NI on the same net as best_lpn chosen
                 * above
                 */
-               best_ni = lnet_find_best_ni_on_spec_net(NULL, peer,
-                                                       best_lpn, md_cpt);
+               best_ni = lnet_find_best_ni_on_spec_net(NULL, peer, best_lpn,
+                                                       msg, md_cpt);
        }
 
        return best_ni;
@@ -2506,11 +2553,12 @@ lnet_select_preferred_best_ni(struct lnet_send_data *sd)
                best_ni =
                  lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer,
                                                sd->sd_best_lpni->lpni_peer_net,
+                                               sd->sd_msg,
                                                sd->sd_md_cpt);
                /* If there is no best_ni we don't have a route */
                if (!best_ni) {
                        CERROR("no path to %s from net %s\n",
-                               libcfs_nid2str(best_lpni->lpni_nid),
+                               libcfs_nidstr(&best_lpni->lpni_nid),
                                libcfs_net2str(best_lpni->lpni_net->net_id));
                        return -EHOSTUNREACH;
                }
@@ -2562,6 +2610,7 @@ lnet_handle_any_local_nmr_dst(struct lnet_send_data *sd)
                sd->sd_best_ni = lnet_find_best_ni_on_spec_net(NULL,
                                                               sd->sd_peer,
                                                               sd->sd_best_lpni->lpni_peer_net,
+                                                              sd->sd_msg,
                                                               sd->sd_md_cpt);
                if (!sd->sd_best_ni) {
                        CERROR("Unable to forward message to %s. No local NI available\n",
@@ -2595,6 +2644,7 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd)
                sd->sd_best_ni =
                  lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer,
                                                sd->sd_best_lpni->lpni_peer_net,
+                                               sd->sd_msg,
                                                sd->sd_md_cpt);
 
                if (!sd->sd_best_ni) {
@@ -2619,6 +2669,7 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd)
         */
        sd->sd_best_ni = lnet_find_best_ni_on_local_net(sd->sd_peer,
                                        sd->sd_md_cpt,
+                                       sd->sd_msg,
                                        lnet_msg_discovery(sd->sd_msg));
        if (sd->sd_best_ni) {
                sd->sd_best_lpni =
@@ -2633,7 +2684,8 @@ lnet_handle_any_mr_dsta(struct lnet_send_data *sd)
                 * network
                 */
                if (sd->sd_best_lpni &&
-                   sd->sd_best_lpni->lpni_nid == the_lnet.ln_loni->ni_nid) {
+                   nid_same(&sd->sd_best_lpni->lpni_nid,
+                            &the_lnet.ln_loni->ni_nid)) {
                        /*
                         * in case we initially started with a routed
                         * destination, let's reset to local
@@ -3168,7 +3220,7 @@ lnet_finalize_expired_responses(void)
                        if (ktime_compare(now, rspt->rspt_deadline) >= 0 ||
                            the_lnet.ln_mt_state == LNET_MT_STATE_SHUTDOWN) {
                                struct lnet_peer_ni *lpni;
-                               lnet_nid_t nid;
+                               struct lnet_nid nid;
 
                                md = lnet_handle2md(&rspt->rspt_mdh);
                                if (!md) {
@@ -3226,7 +3278,7 @@ lnet_finalize_expired_responses(void)
 
                                CDEBUG(D_NET,
                                       "Response timeout: md = %p: nid = %s\n",
-                                      md, libcfs_nid2str(nid));
+                                      md, libcfs_nidstr(&nid));
 
                                /*
                                 * If there is a timeout on the response
@@ -3234,7 +3286,7 @@ lnet_finalize_expired_responses(void)
                                 * value so that we don't use it
                                 */
                                lnet_net_lock(0);
-                               lpni = lnet_find_peer_ni_locked(nid);
+                               lpni = lnet_peer_ni_find_locked(&nid);
                                if (lpni) {
                                        lnet_handle_remote_failure_locked(lpni);
                                        lnet_peer_ni_decref_locked(lpni);
@@ -3408,7 +3460,7 @@ lnet_recover_local_nis(void)
                lnet_net_unlock(0);
 
                CDEBUG(D_NET, "attempting to recover local ni: %s\n",
-                      libcfs_nid2str(ni->ni_nid));
+                      libcfs_nidstr(&ni->ni_nid));
 
                lnet_ni_lock(ni);
                if (!(ni->ni_recovery_state & LNET_NI_RECOVERY_PENDING)) {
@@ -3418,7 +3470,7 @@ lnet_recover_local_nis(void)
                        LIBCFS_ALLOC(ev_info, sizeof(*ev_info));
                        if (!ev_info) {
                                CERROR("out of memory. Can't recover %s\n",
-                                      libcfs_nid2str(ni->ni_nid));
+                                      libcfs_nidstr(&ni->ni_nid));
                                lnet_ni_lock(ni);
                                ni->ni_recovery_state &=
                                  ~LNET_NI_RECOVERY_PENDING;
@@ -3432,7 +3484,8 @@ lnet_recover_local_nis(void)
                         * We'll unlink the mdh in this case below.
                         */
                        LNetInvalidateMDHandle(&ni->ni_ping_mdh);
-                       nid = ni->ni_nid;
+                       /* FIXME need to handle large-addr nid */
+                       nid = lnet_nid_to_nid4(&ni->ni_nid);
 
                        /*
                         * remove the NI from the local queue and drop the
@@ -3673,7 +3726,7 @@ lnet_recover_peer_nis(void)
                        LIBCFS_ALLOC(ev_info, sizeof(*ev_info));
                        if (!ev_info) {
                                CERROR("out of memory. Can't recover %s\n",
-                                      libcfs_nid2str(lpni->lpni_nid));
+                                      libcfs_nidstr(&lpni->lpni_nid));
                                spin_lock(&lpni->lpni_lock);
                                lpni->lpni_state &= ~LNET_PEER_NI_RECOVERY_PENDING;
                                spin_unlock(&lpni->lpni_lock);
@@ -3683,7 +3736,8 @@ lnet_recover_peer_nis(void)
                        /* look at the comments in lnet_recover_local_nis() */
                        mdh = lpni->lpni_recovery_ping_mdh;
                        LNetInvalidateMDHandle(&lpni->lpni_recovery_ping_mdh);
-                       nid = lpni->lpni_nid;
+                       /* FIXME handle large-addr nid */
+                       nid = lnet_nid_to_nid4(&lpni->lpni_nid);
                        lnet_net_lock(0);
                        list_del_init(&lpni->lpni_recovery);
                        lnet_peer_ni_decref_locked(lpni);
@@ -4210,11 +4264,12 @@ lnet_parse_get(struct lnet_ni *ni, struct lnet_msg *msg, int rdma_get)
        lnet_ni_recv(ni, msg->msg_private, NULL, 0, 0, 0, 0);
        msg->msg_receiving = 0;
 
-       rc = lnet_send(ni->ni_nid, msg, msg->msg_from);
+       /* FIXME need to handle large-addr nid */
+       rc = lnet_send(lnet_nid_to_nid4(&ni->ni_nid), msg, msg->msg_from);
        if (rc < 0) {
                /* didn't get as far as lnet_ni_send() */
                CERROR("%s: Unable to send REPLY for GET from %s: %d\n",
-                      libcfs_nid2str(ni->ni_nid),
+                      libcfs_nidstr(&ni->ni_nid),
                       libcfs_id2str(info.mi_id), rc);
 
                lnet_finalize(msg, rc);
@@ -4245,7 +4300,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg)
        if (md == NULL || md->md_threshold == 0 || md->md_me != NULL) {
                CNETERR("%s: Dropping REPLY from %s for %s "
                        "MD %#llx.%#llx\n",
-                       libcfs_nid2str(ni->ni_nid), libcfs_id2str(src),
+                       libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src),
                        (md == NULL) ? "invalid" : "inactive",
                        hdr->msg.reply.dst_wmd.wh_interface_cookie,
                        hdr->msg.reply.dst_wmd.wh_object_cookie);
@@ -4266,7 +4321,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg)
            (md->md_options & LNET_MD_TRUNCATE) == 0) {
                CNETERR("%s: Dropping REPLY from %s length %d "
                        "for MD %#llx would overflow (%d)\n",
-                       libcfs_nid2str(ni->ni_nid), libcfs_id2str(src),
+                       libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src),
                        rlength, hdr->msg.reply.dst_wmd.wh_object_cookie,
                        mlength);
                lnet_res_unlock(cpt);
@@ -4274,7 +4329,7 @@ lnet_parse_reply(struct lnet_ni *ni, struct lnet_msg *msg)
        }
 
        CDEBUG(D_NET, "%s: Reply from %s of length %d/%d into md %#llx\n",
-              libcfs_nid2str(ni->ni_nid), libcfs_id2str(src),
+              libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src),
               mlength, rlength, hdr->msg.reply.dst_wmd.wh_object_cookie);
 
        lnet_msg_attach_md(msg, md, 0, mlength);
@@ -4314,7 +4369,7 @@ lnet_parse_ack(struct lnet_ni *ni, struct lnet_msg *msg)
                /* Don't moan; this is expected */
                CDEBUG(D_NET,
                       "%s: Dropping ACK from %s to %s MD %#llx.%#llx\n",
-                      libcfs_nid2str(ni->ni_nid), libcfs_id2str(src),
+                      libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src),
                       (md == NULL) ? "invalid" : "inactive",
                       hdr->msg.ack.dst_wmd.wh_interface_cookie,
                       hdr->msg.ack.dst_wmd.wh_object_cookie);
@@ -4327,7 +4382,7 @@ lnet_parse_ack(struct lnet_ni *ni, struct lnet_msg *msg)
        }
 
        CDEBUG(D_NET, "%s: ACK from %s into md %#llx\n",
-              libcfs_nid2str(ni->ni_nid), libcfs_id2str(src),
+              libcfs_nidstr(&ni->ni_nid), libcfs_id2str(src),
               hdr->msg.ack.dst_wmd.wh_object_cookie);
 
        lnet_msg_attach_md(msg, md, 0, 0);
@@ -4439,12 +4494,13 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
        dest_pid = le32_to_cpu(hdr->dest_pid);
        payload_length = le32_to_cpu(hdr->payload_length);
 
-       for_me = (ni->ni_nid == dest_nid);
+       /* FIXME handle large-addr nids */
+       for_me = (lnet_nid_to_nid4(&ni->ni_nid) == dest_nid);
        cpt = lnet_cpt_of_nid(from_nid, ni);
 
        CDEBUG(D_NET, "TRACE: %s(%s) <- %s : %s - %s\n",
                libcfs_nid2str(dest_nid),
-               libcfs_nid2str(ni->ni_nid),
+               libcfs_nidstr(&ni->ni_nid),
                libcfs_nid2str(src_nid),
                lnet_msgtyp2str(type),
                (for_me) ? "for me" : "routed");
@@ -4501,7 +4557,7 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
         * or malicious so we chop them off at the knees :) */
 
        if (!for_me) {
-               if (LNET_NIDNET(dest_nid) == LNET_NIDNET(ni->ni_nid)) {
+               if (LNET_NIDNET(dest_nid) == LNET_NID_NET(&ni->ni_nid)) {
                        /* should have gone direct */
                        CERROR("%s, src %s: Bad dest nid %s "
                               "(should have been sent direct)\n",
@@ -4511,7 +4567,7 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
                        return -EPROTO;
                }
 
-               if (lnet_islocalnid(dest_nid)) {
+               if (lnet_islocalnid4(dest_nid)) {
                        /* dest is another local NI; sender should have used
                         * this node's NID on its own network */
                        CERROR("%s, src %s: Bad dest nid %s "
@@ -4552,8 +4608,9 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
                goto drop;
        }
 
+       /* FIXME need to support large-addr nid */
        if (!list_empty(&the_lnet.ln_drop_rules) &&
-           lnet_drop_rule_match(hdr, ni->ni_nid, NULL)) {
+           lnet_drop_rule_match(hdr, lnet_nid_to_nid4(&ni->ni_nid), NULL)) {
                CDEBUG(D_NET,
                       "%s, src %s, dst %s: Dropping %s to simulate silent message loss\n",
                       libcfs_nid2str(from_nid), libcfs_nid2str(src_nid),
@@ -4597,7 +4654,9 @@ lnet_parse(struct lnet_ni *ni, struct lnet_hdr *hdr, lnet_nid_t from_nid,
        }
 
        lnet_net_lock(cpt);
-       lpni = lnet_nid2peerni_locked(from_nid, ni->ni_nid, cpt);
+       /* FIXME support large-addr nid */
+       lpni = lnet_nid2peerni_locked(from_nid, lnet_nid_to_nid4(&ni->ni_nid),
+                                     cpt);
        if (IS_ERR(lpni)) {
                lnet_net_unlock(cpt);
                CERROR("%s, src %s: Dropping %s "
@@ -5022,7 +5081,7 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg)
 
        if (msg == NULL) {
                CERROR("%s: Dropping REPLY from %s: can't allocate msg\n",
-                      libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id));
+                      libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id));
                goto drop;
        }
 
@@ -5033,7 +5092,7 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg)
 
        if (getmd->md_threshold == 0) {
                CERROR("%s: Dropping REPLY from %s for inactive MD %p\n",
-                       libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id),
+                       libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id),
                        getmd);
                lnet_res_unlock(cpt);
                goto drop;
@@ -5042,10 +5101,11 @@ lnet_create_reply_msg(struct lnet_ni *ni, struct lnet_msg *getmsg)
        LASSERT(getmd->md_offset == 0);
 
        CDEBUG(D_NET, "%s: Reply from %s md %p\n",
-              libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id), getmd);
+              libcfs_nidstr(&ni->ni_nid), libcfs_id2str(peer_id), getmd);
 
        /* setup information for lnet_build_msg_event */
-       msg->msg_initiator = getmsg->msg_txpeer->lpni_peer_net->lpn_peer->lp_primary_nid;
+       msg->msg_initiator =
+               lnet_nid_to_nid4(&getmsg->msg_txpeer->lpni_peer_net->lpn_peer->lp_primary_nid);
        msg->msg_from = peer_id.nid;
        msg->msg_type = LNET_MSG_GET; /* flag this msg as an "optimized" GET */
        msg->msg_hdr.src_nid = peer_id.nid;
@@ -5254,7 +5314,8 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
        cpt = lnet_net_lock_current();
 
        while ((ni = lnet_get_next_ni_locked(NULL, ni))) {
-               if (ni->ni_nid == dstnid) {
+               /* FIXME support large-addr nid */
+               if (lnet_nid_to_nid4(&ni->ni_nid) == dstnid) {
                        if (srcnidp != NULL)
                                *srcnidp = dstnid;
                        if (orderp != NULL) {
@@ -5268,7 +5329,7 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
                        return local_nid_dist_zero ? 0 : 1;
                }
 
-               if (!matched_dstnet && LNET_NIDNET(ni->ni_nid) == dstnet) {
+               if (!matched_dstnet && LNET_NID_NET(&ni->ni_nid) == dstnet) {
                        matched_dstnet = true;
                        /* We matched the destination net, but we may have
                         * additional local NIs to inspect.
@@ -5277,7 +5338,8 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
                         * they may be overwritten if we match local NI above.
                         */
                        if (srcnidp)
-                               *srcnidp = ni->ni_nid;
+                               /* FIXME support large-addr nids */
+                               *srcnidp = lnet_nid_to_nid4(&ni->ni_nid);
 
                        if (orderp) {
                                /* Check if ni was originally created in
@@ -5333,7 +5395,8 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
                                net = lnet_get_net_locked(shortest->lr_lnet);
                                LASSERT(net);
                                ni = lnet_get_next_ni_locked(net, NULL);
-                               *srcnidp = ni->ni_nid;
+                               /* FIXME support large-addr nids */
+                               *srcnidp = lnet_nid_to_nid4(&ni->ni_nid);
                        }
                        if (orderp != NULL)
                                *orderp = order;