From fff650726b2b84c0a521933460e71676cca5785d Mon Sep 17 00:00:00 2001 From: James Simmons Date: Fri, 8 Mar 2024 16:54:57 -0500 Subject: [PATCH] LU-13642 lnet: Allow dynamic IP specification Currently you can setup an NI only using the device interface. It is possible that a device interface has more than one IP address. This change updates lnet_net_cmd() to setup an NI using a specific network address. For further reference please read IP specification in LNet https://wiki.whamcloud.com/display/LNet/IP+specification+in+LNet Test-Parameters: trivial testlist=sanity-lnet Change-Id: I2c456790fe9534bbfe02b0330cce73e80318cc1c Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53605 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Chris Horn Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- lnet/doc/lnetctl.8 | 7 ++- lnet/include/lnet/lib-lnet.h | 5 ++- lnet/lnet/api-ni.c | 83 +++++++++++++++++++++-------------- lnet/lnet/config.c | 32 ++++++++------ lnet/lnet/module.c | 2 +- lnet/utils/lnetconfig/liblnetconfig.c | 6 +++ lnet/utils/lnetctl.c | 48 ++++++++++++++++++-- lustre/tests/sanity-lnet.sh | 27 ++++++++++++ 8 files changed, 156 insertions(+), 54 deletions(-) diff --git a/lnet/doc/lnetctl.8 b/lnet/doc/lnetctl.8 index 0d1b712..98a077b 100644 --- a/lnet/doc/lnetctl.8 +++ b/lnet/doc/lnetctl.8 @@ -59,6 +59,9 @@ are optional\. \-\-if: physical interface (e.g. eth0) . .br +\-\-nid: LNet network address to bring up. Cannot be combined with \-\-net. +. +.br \-\-ip2net: specify networks based on IP address patterns . .br @@ -93,7 +96,9 @@ Delete a network interface given the network name\. \-\-net: net name (e.g. tcp0) . .br - +\-\-nid: lnet network address to bring up. Cannot be combined with \-\-net. +. +.br . .TP \fBlnetctl net\fR show diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 4ec0387..48443fa 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -513,8 +513,8 @@ struct lnet_ni * lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface); struct lnet_ni * -lnet_ni_alloc_w_cpt_array(struct lnet_net *net, __u32 *cpts, __u32 ncpts, - char *iface); +lnet_ni_alloc_w_cpt_array(struct lnet_net *net, struct lnet_nid *nid, + u32 *cpts, u32 ncpts, char *iface); int lnet_ni_add_interface(struct lnet_ni *ni, char *iface); static inline int @@ -685,6 +685,7 @@ struct lnet_remotenet *lnet_find_rnet_locked(__u32 net); int lnet_dyn_add_net(struct lnet_ioctl_config_data *conf); int lnet_dyn_del_net(__u32 net); int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net, + struct lnet_nid *nid, struct lnet_ioctl_config_lnd_tunables *tun); int lnet_dyn_del_ni(struct lnet_nid *nid); int lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason); diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 911bdd9..2acdb38 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -3666,6 +3666,7 @@ out: } int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net_id, + struct lnet_nid *nid, struct lnet_ioctl_config_lnd_tunables *tun) { struct lnet_net *net; @@ -3696,8 +3697,8 @@ int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net_id, } } - ni = lnet_ni_alloc_w_cpt_array(net, conf->lic_cpts, conf->lic_ncpts, - conf->lic_ni_intf); + ni = lnet_ni_alloc_w_cpt_array(net, nid, conf->lic_cpts, + conf->lic_ncpts, conf->lic_ni_intf); if (!ni) { lnet_net_free(net); return -ENOMEM; @@ -6038,8 +6039,7 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, } *ni_list = true; } else if (nla_strcmp(settings, "nid") == 0 && - net_id != LNET_NET_ANY && - info->nlhdr->nlmsg_flags & NLM_F_REPLACE) { + net_id != LNET_NET_ANY) { char nidstr[LNET_NIDSTR_SIZE]; settings = nla_next(settings, &rem3); @@ -6061,6 +6061,13 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, GENL_SET_ERR_MSG(info, "unsupported NID"); GOTO(out, rc); } + + if (!(info->nlhdr->nlmsg_flags & NLM_F_REPLACE) && + nid_same(&nid, &LNET_ANY_NID)) { + GENL_SET_ERR_MSG(info, "any NID not supported"); + GOTO(out, rc = -EINVAL); + } + *ni_list = true; } else if (nla_strcmp(settings, "health stats") == 0 && info->nlhdr->nlmsg_flags & NLM_F_REPLACE) { struct nlattr *health; @@ -6195,13 +6202,14 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, } if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) { - if (!strlen(conf->lic_ni_intf)) { + if (nid_same(&nid, &LNET_ANY_NID) && + !strlen(conf->lic_ni_intf)) { GENL_SET_ERR_MSG(info, - "interface is missing"); + "interface / NID is missing"); GOTO(out, rc); } - rc = lnet_dyn_add_ni(conf, net_id, tun); + rc = lnet_dyn_add_ni(conf, net_id, &nid, tun); switch (rc) { case -ENOENT: GENL_SET_ERR_MSG(info, @@ -6220,47 +6228,56 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, } else if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE && healthv != -1) { lnet_ni_set_healthv(&nid, healthv); } else if (!(info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE))) { - struct lnet_net *net; struct lnet_ni *ni; /* delete case */ rc = -ENODEV; - if (!strlen(conf->lic_ni_intf)) { - GENL_SET_ERR_MSG(info, - "interface is missing"); - GOTO(out, rc); - } - - lnet_net_lock(LNET_LOCK_EX); - net = lnet_get_net_locked(net_id); - if (!net) { + if (!strlen(conf->lic_ni_intf) && + nid_same(&nid, &LNET_ANY_NID)) { GENL_SET_ERR_MSG(info, - "LNet net doesn't exist"); - lnet_net_unlock(LNET_LOCK_EX); + "interface / NID is missing"); GOTO(out, rc); } - list_for_each_entry(ni, &net->net_ni_list, - ni_netlist) { - if (!ni->ni_interface || - strcmp(ni->ni_interface, - conf->lic_ni_intf) != 0) - continue; + if (nid_same(&nid, &LNET_ANY_NID)) { + struct lnet_net *net; + bool found = false; - lnet_net_unlock(LNET_LOCK_EX); - rc = lnet_dyn_del_ni(&ni->ni_nid); - if (rc < 0) { + lnet_net_lock(LNET_LOCK_EX); + net = lnet_get_net_locked(net_id); + if (!net) { GENL_SET_ERR_MSG(info, - "cannot del LNet NI"); + "LNet net doesn't exist"); + lnet_net_unlock(LNET_LOCK_EX); GOTO(out, rc); } - break; + + list_for_each_entry(ni, &net->net_ni_list, + ni_netlist) { + if (!ni->ni_interface || + strcmp(ni->ni_interface, + conf->lic_ni_intf) != 0) + continue; + + found = true; + lnet_net_unlock(LNET_LOCK_EX); + rc = lnet_dyn_del_ni(&ni->ni_nid); + break; + } + + if (rc < 0 && !found) { /* will be -ENODEV */ + GENL_SET_ERR_MSG(info, + "interface invalid for deleting LNet NI"); + lnet_net_unlock(LNET_LOCK_EX); + } + } else { + rc = lnet_dyn_del_ni(&nid); } - if (rc < 0) { /* will be -ENODEV */ + if (rc < 0) { GENL_SET_ERR_MSG(info, - "interface invalid for deleting LNet NI"); - lnet_net_unlock(LNET_LOCK_EX); + "cannot del LNet NI"); + GOTO(out, rc); } } out: diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index 4749ab2..e7dece4 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -414,7 +414,7 @@ int lnet_ni_add_interface(struct lnet_ni *ni, char *iface) EXPORT_SYMBOL(lnet_ni_add_interface); static struct lnet_ni * -lnet_ni_alloc_common(struct lnet_net *net, char *iface) +lnet_ni_alloc_common(struct lnet_net *net, struct lnet_nid *nid, char *iface) { struct lnet_tx_queue *tq; struct lnet_ni *ni; @@ -452,9 +452,13 @@ lnet_ni_alloc_common(struct lnet_net *net, char *iface) INIT_LIST_HEAD(&tq->tq_delayed); ni->ni_net = net; - /* LND will fill in the address part of the NID */ - ni->ni_nid.nid_type = LNET_NETTYP(net->net_id); - ni->ni_nid.nid_num = cpu_to_be16(LNET_NETNUM(net->net_id)); + if (nid_same(nid, &LNET_ANY_NID)) { + /* LND will fill in the address part of the NID */ + ni->ni_nid.nid_type = LNET_NETTYP(net->net_id); + ni->ni_nid.nid_num = cpu_to_be16(LNET_NETNUM(net->net_id)); + } else { + ni->ni_nid = *nid; + } /* Store net namespace in which current ni is being created */ if (current->nsproxy && current->nsproxy->net_ns) @@ -487,7 +491,7 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface) struct lnet_ni *ni; int rc; - ni = lnet_ni_alloc_common(net, iface); + ni = lnet_ni_alloc_common(net, &LNET_ANY_NID, iface); if (!ni) return NULL; @@ -523,13 +527,13 @@ failed: } struct lnet_ni * -lnet_ni_alloc_w_cpt_array(struct lnet_net *net, __u32 *cpts, __u32 ncpts, - char *iface) +lnet_ni_alloc_w_cpt_array(struct lnet_net *net, struct lnet_nid *nid, + u32 *cpts, u32 ncpts, char *iface) { struct lnet_ni *ni; int rc; - ni = lnet_ni_alloc_common(net, iface); + ni = lnet_ni_alloc_common(net, nid, iface); if (!ni) return NULL; @@ -1669,7 +1673,7 @@ int lnet_inet_select(struct lnet_ni *ni, return 0; for (if_idx = 0; if_idx < num_ifaces; if_idx++) { - if (ni->ni_interface && + if (ni->ni_interface && strlen(ni->ni_interface) && strcmp(ni->ni_interface, ifaces[if_idx].li_name) != 0) /* not the specified interface */ continue; @@ -1693,15 +1697,15 @@ int lnet_inet_select(struct lnet_ni *ni, if (if_idx < num_ifaces) return if_idx; - if (ni->ni_interface) + if (addr_set) + CERROR("%s: failed to find IP address %s\n", + libcfs_lnd2modname(ni->ni_nid.nid_type), + libcfs_nidstr(&ni->ni_nid)); + else if (ni->ni_interface) CERROR("%s: failed to find interface %s%s%s\n", libcfs_lnd2modname(ni->ni_nid.nid_type), ni->ni_interface, addr_set ? "@" : "", addr_set ? libcfs_nidstr(&ni->ni_nid) : ""); - else - CERROR("%s: failed to find IP address %s\n", - libcfs_lnd2modname(ni->ni_nid.nid_type), - libcfs_nidstr(&ni->ni_nid)); return -EINVAL; } diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index ead3cf5..b58263f 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -152,7 +152,7 @@ lnet_dyn_configure_ni(struct libcfs_ioctl_hdr *hdr) lnet_nid4_to_nid(conf->lic_nid, &nid); net_id = LNET_NID_NET(&nid); - rc = lnet_dyn_add_ni(conf, net_id, tun); + rc = lnet_dyn_add_ni(conf, net_id, &LNET_ANY_NID, tun); } mutex_unlock(&lnet_config_mutex); diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 958b5cb..f350e32 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -320,6 +320,8 @@ int lustre_lnet_parse_interfaces(char *intf_str, return LUSTRE_CFG_RC_BAD_PARAM; while (cur < end) { + char *net; + open_square = strchr(cur, '['); if (open_square != NULL) { close_square = strchr(cur, ']'); @@ -347,6 +349,10 @@ int lustre_lnet_parse_interfaces(char *intf_str, } } + /* Extract net id if its a NID string */ + net = strchr(cur, '@'); + if (net) + nw_descr->nw_id = libcfs_str2net(net + 1); rc = lustre_lnet_add_intf_descr(&nw_descr->nw_intflist, cur, len); if (rc != LUSTRE_CFG_RC_NO_ERR) goto failed; diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index 4ee44b9..d20e2d9 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -145,6 +145,7 @@ command_t net_cmds[] = { {"add", jt_add_ni, 0, "add a network\n" "\t--net: net name (e.g. tcp0)\n" "\t--if: physical interface (e.g. eth0)\n" + "\t--nid: bring up NI based on the address of the specified LNet NID\n" "\t--ip2net: specify networks based on IP address patterns\n" "\t--peer-timeout: time to wait before declaring a peer dead\n" "\t--peer-credits: define the max number of inflight messages\n" @@ -157,6 +158,7 @@ command_t net_cmds[] = { "\t--traffic-class: Traffic class (kfilnd only)\n"}, {"del", jt_del_ni, 0, "delete a network\n" "\t--net: net name (e.g. tcp0)\n" + "\t--nid: shutdown NI based on the address of the specified LNet NID\n" "\t--if: physical interface (e.g. eth0)\n"}, {"show", jt_show_net, 0, "show networks\n" "\t--net: net name (e.g. tcp0) to filter on\n" @@ -2081,9 +2083,9 @@ static int yaml_lnet_config_ni(char *net_id, char *ip2net, goto emitter_error; /* Use NI addresses instead of interface */ - if ((strchr(intf->intf_name, '@') || - strcmp(intf->intf_name, "") == 0) && - flags == NLM_F_REPLACE) { + if (strchr(intf->intf_name, '@') || + (strcmp(intf->intf_name, "") == 0 && +- flags == NLM_F_REPLACE)) { yaml_scalar_event_initialize(&event, NULL, (yaml_char_t *)YAML_STR_TAG, (yaml_char_t *)"nid", @@ -2332,12 +2334,14 @@ static int jt_add_ni(int argc, char **argv) { .name = "conns-per-peer", .has_arg = required_argument, .val = 'm' }, { .name = "net", .has_arg = required_argument, .val = 'n' }, + { .name = "nid", .has_arg = required_argument, .val = 'N' }, { .name = "ip2net", .has_arg = required_argument, .val = 'p' }, { .name = "credits", .has_arg = required_argument, .val = 'r' }, { .name = "cpt", .has_arg = required_argument, .val = 's' }, { .name = "peer-timeout", .has_arg = required_argument, .val = 't' }, { .name = "traffic-class", .has_arg = required_argument, .val = 'T' }, { .name = NULL } }; + bool nid_request = false; char *net_id = NULL; memset(&tunables, 0, sizeof(tunables)); @@ -2399,6 +2403,16 @@ static int jt_add_ni(int argc, char **argv) nw_descr.nw_id = libcfs_str2net(optarg); net_id = optarg; break; + case 'N': + rc = lustre_lnet_parse_interfaces(optarg, &nw_descr); + if (rc != 0) { + cYAML_build_error(-1, -1, "ni", "add", + "bad NID list", + &err_rc); + goto failed; + } + nid_request = true; + break; case 'p': ip2net = optarg; break; @@ -2440,6 +2454,14 @@ static int jt_add_ni(int argc, char **argv) return 0; } } + + if (nid_request) { + if (net_id) { + fprintf(stdout, "--net is invalid with --nid option\n"); + return -EINVAL; + } + net_id = libcfs_net2str(nw_descr.nw_id); + } #ifdef HAVE_KFILND if (auth_key > 0 && LNET_NETTYP(nw_descr.nw_id) == KFILND) { tunables.lt_tun.lnd_tun_u.lnd_kfi.lnd_auth_key = auth_key; @@ -2573,7 +2595,9 @@ static int jt_del_ni(int argc, char **argv) static const struct option long_options[] = { { .name = "net", .has_arg = required_argument, .val = 'n' }, { .name = "if", .has_arg = required_argument, .val = 'i' }, + { .name = "nid", .has_arg = required_argument, .val = 'N' }, { .name = NULL } }; + bool nid_request = false; char *net_id = NULL; rc = check_cmd(net_cmds, "net", "del", 0, argc, argv); @@ -2589,6 +2613,16 @@ static int jt_del_ni(int argc, char **argv) nw_descr.nw_id = libcfs_str2net(optarg); net_id = optarg; break; + case 'N': + rc = lustre_lnet_parse_interfaces(optarg, &nw_descr); + if (rc != 0) { + cYAML_build_error(-1, -1, "ni", "del", + "bad NID list", + &err_rc); + goto out; + } + nid_request = true; + break; case 'i': rc = lustre_lnet_parse_interfaces(optarg, &nw_descr); if (rc != 0) { @@ -2605,6 +2639,14 @@ static int jt_del_ni(int argc, char **argv) } } + if (nid_request) { + if (net_id) { + fprintf(stdout, "--net is invalid with --nid option\n"); + return -EINVAL; + } + net_id = libcfs_net2str(nw_descr.nw_id); + } + rc = yaml_lnet_config_ni(net_id, NULL, &nw_descr, NULL, -1, NULL, LNET_GENL_VERSION, 0); if (rc <= 0) { diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index 78cc861..366099b 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -1486,6 +1486,33 @@ test_109() { } run_test 109 "Add NI using a network interface alias (LU-16859)" +test_110() { + [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" + + cleanup_netns || error "Failed to cleanup netns before test execution" + cleanup_lnet || error "Failed to unload modules before test execution" + + configure_dlc || error "Failed to configure DLC rc = $?" + + LOCAL_ADDR_LIST=$(local_addr_list) + set -- $LOCAL_ADDR_LIST + do_lnetctl net add --nid $2@tcp || + error "Failed to add net tcp for IP $2@tcp" + + $LNETCTL net show --net tcp | grep -q "nid: $2@tcp" || + error "Failed to configure $2@tcp" + + do_lnetctl net del --nid $2@tcp || + error "Failed to del net tcp for IP $2@tcp" + + $LNETCTL net show | grep -q "nid: $2@tcp"&& + error "$2@tcp should have been deleted" + + cleanup_lnet + setup_netns +} +run_test 110 "Add NI using a specific TCP / IP address" + test_200() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" -- 1.8.3.1