From f016fea9a349b450733329cba010634f010b1d0d Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Thu, 31 Mar 2016 16:53:02 -0700 Subject: [PATCH] LU-7734 lnet: handle non-MR peers Add the ability to declare a peer to be non-MR from the DLC interface. By default if a peer is configured from DLC it is assumed to be MR capable, except when the non-mr flag is set. For non-MR peers always use the same NI to communicate with it. If multiple NIs are used to communicate with a non-MR peer the peer will consider that it's talking to different peers which could cause upper layers to be confused. Signed-off-by: Amir Shehata Change-Id: Ie3ec45f5f44fa7d72e3e0335b1383f9c3cc92627 Reviewed-on: http://review.whamcloud.com/19305 Tested-by: Jenkins Reviewed-by: Doug Oucharek Tested-by: Maloo Reviewed-by: Olaf Weber --- lnet/include/lnet/lib-dlc.h | 1 + lnet/include/lnet/lib-lnet.h | 18 +++++++++++++++++- lnet/lnet/api-ni.c | 3 ++- lnet/lnet/lib-move.c | 14 ++++++++++++++ lnet/lnet/peer.c | 7 ++++--- lnet/utils/lnetconfig/liblnetconfig.c | 7 +++++-- lnet/utils/lnetconfig/liblnetconfig.h | 3 ++- lnet/utils/lnetctl.c | 8 ++++++-- 8 files changed, 51 insertions(+), 10 deletions(-) diff --git a/lnet/include/lnet/lib-dlc.h b/lnet/include/lnet/lib-dlc.h index de049d0..016e392 100644 --- a/lnet/include/lnet/lib-dlc.h +++ b/lnet/include/lnet/lib-dlc.h @@ -210,6 +210,7 @@ struct lnet_ioctl_peer_cfg { lnet_nid_t prcfg_key_nid; lnet_nid_t prcfg_cfg_nid; __u32 prcfg_idx; + bool prcfg_mr; char prcfg_bulk[0]; }; diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index dcac6d7..2ef7187 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -815,7 +815,7 @@ struct lnet_peer_net *lnet_peer_get_net_locked(struct lnet_peer *peer, __u32 net_id); bool lnet_peer_is_ni_pref_locked(struct lnet_peer_ni *lpni, struct lnet_ni *ni); -int lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid); +int lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid, bool mr); int lnet_del_peer_ni_from_peer(lnet_nid_t key_nid, lnet_nid_t nid); int lnet_get_peer_info(__u32 idx, lnet_nid_t *primary_nid, lnet_nid_t *nid, struct lnet_peer_ni_credit_info *peer_ni_info); @@ -826,6 +826,22 @@ int lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid, __u32 *peer_rtr_credits, __u32 *peer_min_rtr_credtis, __u32 *peer_tx_qnob); + +static inline __u32 +lnet_get_num_peer_nis(struct lnet_peer *peer) +{ + struct lnet_peer_net *lpn; + struct lnet_peer_ni *lpni; + __u32 count = 0; + + list_for_each_entry(lpn, &peer->lp_peer_nets, lpn_on_peer_list) + list_for_each_entry(lpni, &lpn->lpn_peer_nis, + lpni_on_peer_net_list) + count++; + + return count; +} + static inline bool lnet_is_peer_ni_healthy_locked(struct lnet_peer_ni *lpni) { diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index ecde283..c6b0907 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -2770,7 +2770,8 @@ LNetCtl(unsigned int cmd, void *arg) return -EINVAL; return lnet_add_peer_ni_to_peer(cfg->prcfg_key_nid, - cfg->prcfg_cfg_nid); + cfg->prcfg_cfg_nid, + cfg->prcfg_mr); } case IOC_LIBCFS_DEL_PEER_NI: { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 05cc135..f42857a 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1344,6 +1344,13 @@ again: return -EHOSTUNREACH; } + if (!peer->lp_multi_rail && lnet_get_num_peer_nis(peer) > 1) { + CERROR("peer %s is declared to be non MR capable, " + "yet configured with more than one NID\n", + libcfs_nid2str(dst_nid)); + return -EINVAL; + } + /* * STEP 1: first jab at determineing best_ni * if src_nid is explicitly specified, then best_ni is already @@ -1542,6 +1549,13 @@ set_ni: */ best_ni->ni_seq++; + /* + * if the peer is not MR capable, then we should always send to it + * using the first NI in the NET we determined. + */ + if (!peer->lp_multi_rail && local_net != NULL) + best_ni = lnet_net2ni_locked(local_net->net_id, cpt); + if (!best_ni) { lnet_net_unlock(cpt); LCONSOLE_WARN("No local ni found to send from to %s\n", diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index 8e5cdbb..c9e93c5 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -467,6 +467,7 @@ lnet_build_peer_hierarchy(struct lnet_peer_ni *lpni) peer_net->lpn_peer = peer; lpni->lpni_peer_net = peer_net; peer->lp_primary_nid = lpni->lpni_nid; + peer->lp_multi_rail = false; list_add_tail(&peer_net->lpn_on_peer_list, &peer->lp_peer_nets); list_add_tail(&lpni->lpni_on_peer_net_list, &peer_net->lpn_peer_nis); list_add_tail(&peer->lp_on_lnet_peer_list, &the_lnet.ln_peers); @@ -491,7 +492,7 @@ lnet_peer_get_net_locked(struct lnet_peer *peer, __u32 net_id) * is unique */ int -lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid) +lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid, bool mr) { struct lnet_peer_ni *lpni, *lpni2; struct lnet_peer *peer; @@ -524,14 +525,14 @@ lnet_add_peer_ni_to_peer(lnet_nid_t key_nid, lnet_nid_t nid) return -EINVAL; } peer = lpni->lpni_peer_net->lpn_peer; - peer->lp_multi_rail = true; + peer->lp_multi_rail = mr; lnet_peer_ni_decref_locked(lpni); lnet_net_unlock(cpt2); } else { lnet_net_lock(LNET_LOCK_EX); rc = lnet_nid2peerni_locked(&lpni, nid, LNET_LOCK_EX); if (rc == 0) { - lpni->lpni_peer_net->lpn_peer->lp_multi_rail = true; + lpni->lpni_peer_net->lpn_peer->lp_multi_rail = mr; lnet_peer_ni_decref_locked(lpni); } lnet_net_unlock(LNET_LOCK_EX); diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index aa23ce0..335072d 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -339,7 +339,7 @@ static int dispatch_peer_ni_cmd(lnet_nid_t knid, lnet_nid_t nid, __u32 cmd, return rc; } -int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no, +int lustre_lnet_config_peer_nid(char *knid, char **nid, bool mr, int seq_no, struct cYAML **err_rc) { struct lnet_ioctl_peer_cfg data; @@ -370,6 +370,7 @@ int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no, snprintf(err_str, sizeof(err_str), "\"Success\""); LIBCFS_IOC_INIT_V2(data, prcfg_hdr); + data.prcfg_mr = mr; if (nids[0] == LNET_NID_ANY) { rc = dispatch_peer_ni_cmd(LNET_NID_ANY, key_nid, IOC_LIBCFS_ADD_PEER_NI, @@ -2657,7 +2658,7 @@ static int handle_yaml_config_peer(struct cYAML *tree, struct cYAML **show_rc, { char **nids = NULL; int num, rc; - struct cYAML *seq_no, *key_nid; + struct cYAML *seq_no, *key_nid, *non_mr; num = yaml_copy_peer_nids(tree, &nids); if (num < 0) @@ -2665,9 +2666,11 @@ static int handle_yaml_config_peer(struct cYAML *tree, struct cYAML **show_rc, seq_no = cYAML_get_object_item(tree, "seq_no"); key_nid = cYAML_get_object_item(tree, "key_nid"); + non_mr = cYAML_get_object_item(tree, "non_mr"); rc = lustre_lnet_config_peer_nid((key_nid) ? key_nid->cy_valuestring : NULL, nids, + (non_mr) ? false : true, (seq_no) ? seq_no->cy_valueint : -1, err_rc); diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index 4491d14..54bcc94 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -271,10 +271,11 @@ int lustre_lnet_show_stats(int seq_no, struct cYAML **show_rc, * * knid - Key NID of the peer * nid - list of nids to add + * mr - true if this peer is MR capable. * seq_no - sequence number of the command * err_rc - YAML strucutre of the resultant return code. */ -int lustre_lnet_config_peer_nid(char *knid, char **nid, int seq_no, +int lustre_lnet_config_peer_nid(char *knid, char **nid, bool mr, int seq_no, struct cYAML **err_rc); /* diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index dea8c9b..cbbd33e 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -1080,11 +1080,13 @@ static int jt_add_peer_nid(int argc, char **argv) int idx = 0; struct cYAML *err_rc = NULL; int rc, opt; + bool non_mr = false; - const char *const short_options = "k:n:h"; + const char *const short_options = "k:n:mh"; const struct option long_options[] = { { "key_nid", 1, NULL, 'k' }, { "nid", 1, NULL, 'n' }, + { "non_mr", 1, NULL, 'm'}, { "help", 0, NULL, 'h' }, { NULL, 0, NULL, 0 }, }; @@ -1114,6 +1116,8 @@ static int jt_add_peer_nid(int argc, char **argv) strncpy(nid[idx], optarg, strlen(optarg)); idx++; break; + case 'm': + non_mr = true; case 'h': print_help(peer_cmds, "peer", "add"); return 0; @@ -1122,7 +1126,7 @@ static int jt_add_peer_nid(int argc, char **argv) } } - rc = lustre_lnet_config_peer_nid(key_nid, nid, -1, &err_rc); + rc = lustre_lnet_config_peer_nid(key_nid, nid, !non_mr, -1, &err_rc); failed: idx = 0; -- 1.8.3.1