LU-7734 lnet: handle non-MR peers

Add the ability to declare a peer to be non-MR from the DLC
interface. By default if a peer is configured from DLC it is
assumed to be MR capable, except when the non-mr flag is set.

For non-MR peers always use the same NI to communicate with it.
If multiple NIs are used to communicate with a non-MR peer the
peer will consider that it's talking to different peers which could
cause upper layers to be confused.

Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: Ie3ec45f5f44fa7d72e3e0335b1383f9c3cc92627
Reviewed-on: http://review.whamcloud.com/19305
Tested-by: Jenkins
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>

diff --git a/lnet/include/lnet/lib-dlc.h b/lnet/include/lnet/lib-dlc.h
index de049d0..016e392 100644 (file)
--- a/lnet/include/lnet/lib-dlc.h
+++ b/lnet/include/lnet/lib-dlc.h
@@ -210,6 +210,7 @@ struct lnet_ioctl_peer_cfg {
        lnet_nid_t prcfg_key_nid;
        lnet_nid_t prcfg_cfg_nid;
        __u32 prcfg_idx;
+       bool prcfg_mr;
        char prcfg_bulk[0];
 };
 

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index dcac6d7..2ef7187 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -815,7 +815,7 @@ struct lnet_peer_net *lnet_peer_get_net_locked(struct lnet_peer *peer,
                                               __u32 net_id);
 bool lnet_peer_is_ni_pref_locked(struct lnet_peer_ni *lpni,
                                 struct lnet_ni *ni);
-int lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid);
+int lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid, bool mr);
 int lnet_del_peer_ni_from_peer(lnet_nid_t key_nid, lnet_nid_t nid);
 int lnet_get_peer_info(__u32 idx, lnet_nid_t *primary_nid, lnet_nid_t *nid,
                       struct lnet_peer_ni_credit_info *peer_ni_info);
@@ -826,6 +826,22 @@ int lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid,
                          __u32 *peer_rtr_credits, __u32 *peer_min_rtr_credtis,
                          __u32 *peer_tx_qnob);
 
+
+static inline __u32
+lnet_get_num_peer_nis(struct lnet_peer *peer)
+{
+       struct lnet_peer_net *lpn;
+       struct lnet_peer_ni *lpni;
+       __u32 count = 0;
+
+       list_for_each_entry(lpn, &peer->lp_peer_nets, lpn_on_peer_list)
+               list_for_each_entry(lpni, &lpn->lpn_peer_nis,
+                                   lpni_on_peer_net_list)
+                       count++;
+
+       return count;
+}
+
 static inline bool
 lnet_is_peer_ni_healthy_locked(struct lnet_peer_ni *lpni)
 {

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index ecde283..c6b0907 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -2770,7 +2770,8 @@ LNetCtl(unsigned int cmd, void *arg)
                        return -EINVAL;
 
                return lnet_add_peer_ni_to_peer(cfg->prcfg_key_nid,
-                                               cfg->prcfg_cfg_nid);
+                                               cfg->prcfg_cfg_nid,
+                                               cfg->prcfg_mr);
        }
 
        case IOC_LIBCFS_DEL_PEER_NI: {

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index 05cc135..f42857a 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -1344,6 +1344,13 @@ again:
                return -EHOSTUNREACH;
        }
 
+       if (!peer->lp_multi_rail && lnet_get_num_peer_nis(peer) > 1) {
+               CERROR("peer %s is declared to be non MR capable, "
+                      "yet configured with more than one NID\n",
+                      libcfs_nid2str(dst_nid));
+               return -EINVAL;
+       }
+
        /*
         * STEP 1: first jab at determineing best_ni
         * if src_nid is explicitly specified, then best_ni is already
@@ -1542,6 +1549,13 @@ set_ni:
         */
        best_ni->ni_seq++;
 
+       /*
+        * if the peer is not MR capable, then we should always send to it
+        * using the first NI in the NET we determined.
+        */
+       if (!peer->lp_multi_rail && local_net != NULL)
+               best_ni = lnet_net2ni_locked(local_net->net_id, cpt);
+
        if (!best_ni) {
                lnet_net_unlock(cpt);
                LCONSOLE_WARN("No local ni found to send from to %s\n",

diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c
index 8e5cdbb..c9e93c5 100644 (file)
--- a/lnet/lnet/peer.c
+++ b/lnet/lnet/peer.c
@@ -467,6 +467,7 @@ lnet_build_peer_hierarchy(struct lnet_peer_ni *lpni)
        peer_net->lpn_peer = peer;
        lpni->lpni_peer_net = peer_net;
        peer->lp_primary_nid = lpni->lpni_nid;
+       peer->lp_multi_rail = false;
        list_add_tail(&peer_net->lpn_on_peer_list, &peer->lp_peer_nets);
        list_add_tail(&lpni->lpni_on_peer_net_list, &peer_net->lpn_peer_nis);
        list_add_tail(&peer->lp_on_lnet_peer_list, &the_lnet.ln_peers);
@@ -491,7 +492,7 @@ lnet_peer_get_net_locked(struct lnet_peer *peer, __u32 net_id)
  * is unique
  */
 int
-lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid)
+lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid, bool mr)
 {
        struct lnet_peer_ni *lpni, *lpni2;
        struct lnet_peer *peer;
@@ -524,14 +525,14 @@ lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid)
                        return -EINVAL;
                }
                peer = lpni->lpni_peer_net->lpn_peer;
-               peer->lp_multi_rail = true;
+               peer->lp_multi_rail = mr;
                lnet_peer_ni_decref_locked(lpni);
                lnet_net_unlock(cpt2);
        } else {
                lnet_net_lock(LNET_LOCK_EX);
                rc = lnet_nid2peerni_locked(&lpni, nid, LNET_LOCK_EX);
                if (rc == 0) {
-                       lpni->lpni_peer_net->lpn_peer->lp_multi_rail = true;
+                       lpni->lpni_peer_net->lpn_peer->lp_multi_rail = mr;
                        lnet_peer_ni_decref_locked(lpni);
                }
                lnet_net_unlock(LNET_LOCK_EX);

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index aa23ce0..335072d 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -339,7 +339,7 @@ static int dispatch_peer_ni_cmd(lnet_nid_t knid, lnet_nid_t nid, __u32 cmd,
        return rc;
 }
 
-int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no,
+int lustre_lnet_config_peer_nid(char *knid, char **nid, bool mr, int seq_no,
                                struct cYAML **err_rc)
 {
        struct lnet_ioctl_peer_cfg data;
@@ -370,6 +370,7 @@ int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no,
        snprintf(err_str, sizeof(err_str), "\"Success\"");
 
        LIBCFS_IOC_INIT_V2(data, prcfg_hdr);
+       data.prcfg_mr = mr;
        if (nids[0] == LNET_NID_ANY) {
                rc = dispatch_peer_ni_cmd(LNET_NID_ANY, key_nid,
                                          IOC_LIBCFS_ADD_PEER_NI,
@@ -2657,7 +2658,7 @@ static int handle_yaml_config_peer(struct cYAML *tree, struct cYAML **show_rc,
 {
        char **nids = NULL;
        int num, rc;
-       struct cYAML *seq_no, *key_nid;
+       struct cYAML *seq_no, *key_nid, *non_mr;
 
        num = yaml_copy_peer_nids(tree, &nids);
        if (num < 0)
@@ -2665,9 +2666,11 @@ static int handle_yaml_config_peer(struct cYAML *tree, struct cYAML **show_rc,
 
        seq_no = cYAML_get_object_item(tree, "seq_no");
        key_nid = cYAML_get_object_item(tree, "key_nid");
+       non_mr = cYAML_get_object_item(tree, "non_mr");
 
        rc = lustre_lnet_config_peer_nid((key_nid) ? key_nid->cy_valuestring : NULL,
                                         nids,
+                                        (non_mr) ? false : true,
                                         (seq_no) ? seq_no->cy_valueint : -1,
                                         err_rc);
 

diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h
index 4491d14..54bcc94 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.h
+++ b/lnet/utils/lnetconfig/liblnetconfig.h
@@ -271,10 +271,11 @@ int lustre_lnet_show_stats(int seq_no, struct cYAML **show_rc,
  *
  *     knid - Key NID of the peer
  *     nid - list of nids to add
+ *     mr - true if this peer is MR capable.
  *     seq_no - sequence number of the command
  *     err_rc - YAML strucutre of the resultant return code.
  */
-int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no,
+int lustre_lnet_config_peer_nid(char *knid, char **nid, bool mr, int seq_no,
                                struct cYAML **err_rc);
 
 /*

diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c
index dea8c9b..cbbd33e 100644 (file)
--- a/lnet/utils/lnetctl.c
+++ b/lnet/utils/lnetctl.c
@@ -1080,11 +1080,13 @@ static int jt_add_peer_nid(int argc, char **argv)
        int idx = 0;
        struct cYAML *err_rc = NULL;
        int rc, opt;
+       bool non_mr = false;
 
-       const char *const short_options = "k:n:h";
+       const char *const short_options = "k:n:mh";
        const struct option long_options[] = {
                { "key_nid", 1, NULL, 'k' },
                { "nid", 1, NULL, 'n' },
+               { "non_mr", 1, NULL, 'm'},
                { "help", 0, NULL, 'h' },
                { NULL, 0, NULL, 0 },
        };
@@ -1114,6 +1116,8 @@ static int jt_add_peer_nid(int argc, char **argv)
                        strncpy(nid[idx], optarg, strlen(optarg));
                        idx++;
                        break;
+               case 'm':
+                       non_mr = true;
                case 'h':
                        print_help(peer_cmds, "peer", "add");
                        return 0;
@@ -1122,7 +1126,7 @@ static int jt_add_peer_nid(int argc, char **argv)
                }
        }
 
-       rc = lustre_lnet_config_peer_nid(key_nid, nid, -1, &err_rc);
+       rc = lustre_lnet_config_peer_nid(key_nid, nid, !non_mr, -1, &err_rc);
 
 failed:
        idx = 0;