LU-13642 lnet: Allow dynamic IP specification

Currently you can setup an NI only using the device interface.
It is possible that a device interface has more than one IP
address. This change updates lnet_net_cmd() to setup an NI using
a specific network address.

For further reference please read

IP specification in LNet
https://wiki.whamcloud.com/display/LNet/IP+specification+in+LNet

Test-Parameters: trivial testlist=sanity-lnet
Change-Id: I2c456790fe9534bbfe02b0330cce73e80318cc1c
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53605
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/doc/lnetctl.8 b/lnet/doc/lnetctl.8
index 0d1b712..98a077b 100644 (file)
--- a/lnet/doc/lnetctl.8
+++ b/lnet/doc/lnetctl.8
@@ -59,6 +59,9 @@ are optional\.
 \-\-if: physical interface (e.g. eth0)
 .
 .br
+\-\-nid: LNet network address to bring up. Cannot be combined with \-\-net.
+.
+.br
 \-\-ip2net: specify networks based on IP address patterns
 .
 .br
@@ -93,7 +96,9 @@ Delete a network interface given the network name\.
 \-\-net: net name (e.g. tcp0)
 .
 .br
-
+\-\-nid: lnet network address to bring up. Cannot be combined with \-\-net.
+.
+.br
 .
 .TP
 \fBlnetctl net\fR show

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 4ec0387..48443fa 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -513,8 +513,8 @@ struct lnet_ni *
 lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el,
              char *iface);
 struct lnet_ni *
-lnet_ni_alloc_w_cpt_array(struct lnet_net *net, __u32 *cpts, __u32 ncpts,
-                         char *iface);
+lnet_ni_alloc_w_cpt_array(struct lnet_net *net, struct lnet_nid *nid,
+                         u32 *cpts, u32 ncpts, char *iface);
 int lnet_ni_add_interface(struct lnet_ni *ni, char *iface);
 
 static inline int
@@ -685,6 +685,7 @@ struct lnet_remotenet *lnet_find_rnet_locked(__u32 net);
 int lnet_dyn_add_net(struct lnet_ioctl_config_data *conf);
 int lnet_dyn_del_net(__u32 net);
 int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net,
+                   struct lnet_nid *nid,
                    struct lnet_ioctl_config_lnd_tunables *tun);
 int lnet_dyn_del_ni(struct lnet_nid *nid);
 int lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason);

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 911bdd9..2acdb38 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -3666,6 +3666,7 @@ out:
 }
 
 int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net_id,
+                   struct lnet_nid *nid,
                    struct lnet_ioctl_config_lnd_tunables *tun)
 {
        struct lnet_net *net;
@@ -3696,8 +3697,8 @@ int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf, u32 net_id,
                }
        }
 
-       ni = lnet_ni_alloc_w_cpt_array(net, conf->lic_cpts, conf->lic_ncpts,
-                                      conf->lic_ni_intf);
+       ni = lnet_ni_alloc_w_cpt_array(net, nid, conf->lic_cpts,
+                                      conf->lic_ncpts, conf->lic_ni_intf);
        if (!ni) {
                lnet_net_free(net);
                return -ENOMEM;
@@ -6038,8 +6039,7 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info,
                        }
                        *ni_list = true;
                } else if (nla_strcmp(settings, "nid") == 0 &&
-                          net_id != LNET_NET_ANY &&
-                          info->nlhdr->nlmsg_flags & NLM_F_REPLACE) {
+                          net_id != LNET_NET_ANY) {
                        char nidstr[LNET_NIDSTR_SIZE];
 
                        settings = nla_next(settings, &rem3);
@@ -6061,6 +6061,13 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info,
                                GENL_SET_ERR_MSG(info, "unsupported NID");
                                GOTO(out, rc);
                        }
+
+                       if (!(info->nlhdr->nlmsg_flags & NLM_F_REPLACE) &&
+                            nid_same(&nid, &LNET_ANY_NID)) {
+                               GENL_SET_ERR_MSG(info, "any NID not supported");
+                               GOTO(out, rc = -EINVAL);
+                       }
+                       *ni_list = true;
                } else if (nla_strcmp(settings, "health stats") == 0 &&
                           info->nlhdr->nlmsg_flags & NLM_F_REPLACE) {
                        struct nlattr *health;
@@ -6195,13 +6202,14 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info,
        }
 
        if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
-               if (!strlen(conf->lic_ni_intf)) {
+               if (nid_same(&nid, &LNET_ANY_NID) &&
+                   !strlen(conf->lic_ni_intf)) {
                        GENL_SET_ERR_MSG(info,
-                                        "interface is missing");
+                                        "interface / NID is missing");
                        GOTO(out, rc);
                }
 
-               rc = lnet_dyn_add_ni(conf, net_id, tun);
+               rc = lnet_dyn_add_ni(conf, net_id, &nid, tun);
                switch (rc) {
                case -ENOENT:
                        GENL_SET_ERR_MSG(info,
@@ -6220,47 +6228,56 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info,
        } else if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE && healthv != -1) {
                lnet_ni_set_healthv(&nid, healthv);
        } else if (!(info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE))) {
-               struct lnet_net *net;
                struct lnet_ni *ni;
 
                /* delete case */
                rc = -ENODEV;
-               if (!strlen(conf->lic_ni_intf)) {
-                       GENL_SET_ERR_MSG(info,
-                                        "interface is missing");
-                       GOTO(out, rc);
-               }
-
-               lnet_net_lock(LNET_LOCK_EX);
-               net = lnet_get_net_locked(net_id);
-               if (!net) {
+               if (!strlen(conf->lic_ni_intf) &&
+                   nid_same(&nid, &LNET_ANY_NID)) {
                        GENL_SET_ERR_MSG(info,
-                                        "LNet net doesn't exist");
-                       lnet_net_unlock(LNET_LOCK_EX);
+                                        "interface / NID is missing");
                        GOTO(out, rc);
                }
 
-               list_for_each_entry(ni, &net->net_ni_list,
-                                   ni_netlist) {
-                       if (!ni->ni_interface ||
-                           strcmp(ni->ni_interface,
-                                 conf->lic_ni_intf) != 0)
-                               continue;
+               if (nid_same(&nid, &LNET_ANY_NID)) {
+                       struct lnet_net *net;
+                       bool found = false;
 
-                       lnet_net_unlock(LNET_LOCK_EX);
-                       rc = lnet_dyn_del_ni(&ni->ni_nid);
-                       if (rc < 0) {
+                       lnet_net_lock(LNET_LOCK_EX);
+                       net = lnet_get_net_locked(net_id);
+                       if (!net) {
                                GENL_SET_ERR_MSG(info,
-                                                "cannot del LNet NI");
+                                                "LNet net doesn't exist");
+                               lnet_net_unlock(LNET_LOCK_EX);
                                GOTO(out, rc);
                        }
-                       break;
+
+                       list_for_each_entry(ni, &net->net_ni_list,
+                                           ni_netlist) {
+                               if (!ni->ni_interface ||
+                                   strcmp(ni->ni_interface,
+                                         conf->lic_ni_intf) != 0)
+                                       continue;
+
+                               found = true;
+                               lnet_net_unlock(LNET_LOCK_EX);
+                               rc = lnet_dyn_del_ni(&ni->ni_nid);
+                               break;
+                       }
+
+                       if (rc < 0 && !found) { /* will be -ENODEV */
+                               GENL_SET_ERR_MSG(info,
+                                                "interface invalid for deleting LNet NI");
+                               lnet_net_unlock(LNET_LOCK_EX);
+                       }
+               } else {
+                       rc = lnet_dyn_del_ni(&nid);
                }
 
-               if (rc < 0) { /* will be -ENODEV */
+               if (rc < 0) {
                        GENL_SET_ERR_MSG(info,
-                                        "interface invalid for deleting LNet NI");
-                       lnet_net_unlock(LNET_LOCK_EX);
+                                        "cannot del LNet NI");
+                       GOTO(out, rc);
                }
        }
 out:

diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c
index 4749ab2..e7dece4 100644 (file)
--- a/lnet/lnet/config.c
+++ b/lnet/lnet/config.c
@@ -414,7 +414,7 @@ int lnet_ni_add_interface(struct lnet_ni *ni, char *iface)
 EXPORT_SYMBOL(lnet_ni_add_interface);
 
 static struct lnet_ni *
-lnet_ni_alloc_common(struct lnet_net *net, char *iface)
+lnet_ni_alloc_common(struct lnet_net *net, struct lnet_nid *nid, char *iface)
 {
        struct lnet_tx_queue    *tq;
        struct lnet_ni          *ni;
@@ -452,9 +452,13 @@ lnet_ni_alloc_common(struct lnet_net *net, char *iface)
                INIT_LIST_HEAD(&tq->tq_delayed);
 
        ni->ni_net = net;
-       /* LND will fill in the address part of the NID */
-       ni->ni_nid.nid_type = LNET_NETTYP(net->net_id);
-       ni->ni_nid.nid_num = cpu_to_be16(LNET_NETNUM(net->net_id));
+       if (nid_same(nid, &LNET_ANY_NID)) {
+               /* LND will fill in the address part of the NID */
+               ni->ni_nid.nid_type = LNET_NETTYP(net->net_id);
+               ni->ni_nid.nid_num = cpu_to_be16(LNET_NETNUM(net->net_id));
+       } else {
+               ni->ni_nid = *nid;
+       }
 
        /* Store net namespace in which current ni is being created */
        if (current->nsproxy && current->nsproxy->net_ns)
@@ -487,7 +491,7 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
        struct lnet_ni          *ni;
        int                     rc;
 
-       ni = lnet_ni_alloc_common(net, iface);
+       ni = lnet_ni_alloc_common(net, &LNET_ANY_NID, iface);
        if (!ni)
                return NULL;
 
@@ -523,13 +527,13 @@ failed:
 }
 
 struct lnet_ni *
-lnet_ni_alloc_w_cpt_array(struct lnet_net *net, __u32 *cpts, __u32 ncpts,
-                         char *iface)
+lnet_ni_alloc_w_cpt_array(struct lnet_net *net, struct lnet_nid *nid,
+                         u32 *cpts, u32 ncpts, char *iface)
 {
        struct lnet_ni          *ni;
        int                     rc;
 
-       ni = lnet_ni_alloc_common(net, iface);
+       ni = lnet_ni_alloc_common(net, nid, iface);
        if (!ni)
                return NULL;
 
@@ -1669,7 +1673,7 @@ int lnet_inet_select(struct lnet_ni *ni,
                return 0;
 
        for (if_idx = 0; if_idx < num_ifaces; if_idx++) {
-               if (ni->ni_interface &&
+               if (ni->ni_interface && strlen(ni->ni_interface) &&
                        strcmp(ni->ni_interface, ifaces[if_idx].li_name) != 0)
                        /* not the specified interface */
                        continue;
@@ -1693,15 +1697,15 @@ int lnet_inet_select(struct lnet_ni *ni,
        if (if_idx < num_ifaces)
                return if_idx;
 
-       if (ni->ni_interface)
+       if (addr_set)
+               CERROR("%s: failed to find IP address %s\n",
+                      libcfs_lnd2modname(ni->ni_nid.nid_type),
+                      libcfs_nidstr(&ni->ni_nid));
+       else if (ni->ni_interface)
                CERROR("%s: failed to find interface %s%s%s\n",
                       libcfs_lnd2modname(ni->ni_nid.nid_type),
                       ni->ni_interface, addr_set ? "@" : "",
                       addr_set ? libcfs_nidstr(&ni->ni_nid) : "");
-       else
-               CERROR("%s: failed to find IP address %s\n",
-                      libcfs_lnd2modname(ni->ni_nid.nid_type),
-                      libcfs_nidstr(&ni->ni_nid));
 
        return -EINVAL;
 }

diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c
index ead3cf5..b58263f 100644 (file)
--- a/lnet/lnet/module.c
+++ b/lnet/lnet/module.c
@@ -152,7 +152,7 @@ lnet_dyn_configure_ni(struct libcfs_ioctl_hdr *hdr)
 
                lnet_nid4_to_nid(conf->lic_nid, &nid);
                net_id = LNET_NID_NET(&nid);
-               rc = lnet_dyn_add_ni(conf, net_id, tun);
+               rc = lnet_dyn_add_ni(conf, net_id, &LNET_ANY_NID, tun);
        }
        mutex_unlock(&lnet_config_mutex);
 

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index 958b5cb..f350e32 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -320,6 +320,8 @@ int lustre_lnet_parse_interfaces(char *intf_str,
                return LUSTRE_CFG_RC_BAD_PARAM;
 
        while (cur < end) {
+               char *net;
+
                open_square = strchr(cur, '[');
                if (open_square != NULL) {
                        close_square = strchr(cur, ']');
@@ -347,6 +349,10 @@ int lustre_lnet_parse_interfaces(char *intf_str,
                        }
                }
 
+               /* Extract net id if its a NID string */
+               net = strchr(cur, '@');
+               if (net)
+                       nw_descr->nw_id = libcfs_str2net(net + 1);
                rc = lustre_lnet_add_intf_descr(&nw_descr->nw_intflist, cur, len);
                if (rc != LUSTRE_CFG_RC_NO_ERR)
                        goto failed;

diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c
index 4ee44b9..d20e2d9 100644 (file)
--- a/lnet/utils/lnetctl.c
+++ b/lnet/utils/lnetctl.c
@@ -145,6 +145,7 @@ command_t net_cmds[] = {
        {"add", jt_add_ni, 0, "add a network\n"
         "\t--net: net name (e.g. tcp0)\n"
         "\t--if: physical interface (e.g. eth0)\n"
+        "\t--nid: bring up NI based on the address of the specified LNet NID\n"
         "\t--ip2net: specify networks based on IP address patterns\n"
         "\t--peer-timeout: time to wait before declaring a peer dead\n"
         "\t--peer-credits: define the max number of inflight messages\n"
@@ -157,6 +158,7 @@ command_t net_cmds[] = {
         "\t--traffic-class: Traffic class (kfilnd only)\n"},
        {"del", jt_del_ni, 0, "delete a network\n"
         "\t--net: net name (e.g. tcp0)\n"
+        "\t--nid: shutdown NI based on the address of the specified LNet NID\n"
         "\t--if: physical interface (e.g. eth0)\n"},
        {"show", jt_show_net, 0, "show networks\n"
         "\t--net: net name (e.g. tcp0) to filter on\n"
@@ -2081,9 +2083,9 @@ static int yaml_lnet_config_ni(char *net_id, char *ip2net,
                        goto emitter_error;
 
                /* Use NI addresses instead of interface */
-               if ((strchr(intf->intf_name, '@') ||
-                    strcmp(intf->intf_name, "<?>") == 0) &&
-                   flags == NLM_F_REPLACE) {
+               if (strchr(intf->intf_name, '@') ||
+                   (strcmp(intf->intf_name, "<?>") == 0 &&
+-                    flags == NLM_F_REPLACE)) {
                        yaml_scalar_event_initialize(&event, NULL,
                                                     (yaml_char_t *)YAML_STR_TAG,
                                                     (yaml_char_t *)"nid",
@@ -2332,12 +2334,14 @@ static int jt_add_ni(int argc, char **argv)
        { .name = "conns-per-peer",
                                  .has_arg = required_argument, .val = 'm' },
        { .name = "net",          .has_arg = required_argument, .val = 'n' },
+       { .name = "nid",          .has_arg = required_argument, .val = 'N' },
        { .name = "ip2net",       .has_arg = required_argument, .val = 'p' },
        { .name = "credits",      .has_arg = required_argument, .val = 'r' },
        { .name = "cpt",          .has_arg = required_argument, .val = 's' },
        { .name = "peer-timeout", .has_arg = required_argument, .val = 't' },
        { .name = "traffic-class", .has_arg = required_argument, .val = 'T' },
        { .name = NULL } };
+       bool nid_request = false;
        char *net_id = NULL;
 
        memset(&tunables, 0, sizeof(tunables));
@@ -2399,6 +2403,16 @@ static int jt_add_ni(int argc, char **argv)
                        nw_descr.nw_id = libcfs_str2net(optarg);
                        net_id = optarg;
                        break;
+               case 'N':
+                       rc = lustre_lnet_parse_interfaces(optarg, &nw_descr);
+                       if (rc != 0) {
+                               cYAML_build_error(-1, -1, "ni", "add",
+                                               "bad NID list",
+                                               &err_rc);
+                               goto failed;
+                       }
+                       nid_request = true;
+                       break;
                case 'p':
                        ip2net = optarg;
                        break;
@@ -2440,6 +2454,14 @@ static int jt_add_ni(int argc, char **argv)
                        return 0;
                }
        }
+
+       if (nid_request) {
+               if (net_id) {
+                       fprintf(stdout, "--net is invalid with --nid option\n");
+                       return -EINVAL;
+               }
+               net_id = libcfs_net2str(nw_descr.nw_id);
+       }
 #ifdef HAVE_KFILND
        if (auth_key > 0 && LNET_NETTYP(nw_descr.nw_id) == KFILND) {
                tunables.lt_tun.lnd_tun_u.lnd_kfi.lnd_auth_key = auth_key;
@@ -2573,7 +2595,9 @@ static int jt_del_ni(int argc, char **argv)
        static const struct option long_options[] = {
        { .name = "net",        .has_arg = required_argument,   .val = 'n' },
        { .name = "if",         .has_arg = required_argument,   .val = 'i' },
+       { .name = "nid",        .has_arg = required_argument,   .val = 'N' },
        { .name = NULL } };
+       bool nid_request = false;
        char *net_id = NULL;
 
        rc = check_cmd(net_cmds, "net", "del", 0, argc, argv);
@@ -2589,6 +2613,16 @@ static int jt_del_ni(int argc, char **argv)
                        nw_descr.nw_id = libcfs_str2net(optarg);
                        net_id = optarg;
                        break;
+               case 'N':
+                       rc = lustre_lnet_parse_interfaces(optarg, &nw_descr);
+                       if (rc != 0) {
+                               cYAML_build_error(-1, -1, "ni", "del",
+                                                 "bad NID list",
+                                                 &err_rc);
+                               goto out;
+                       }
+                       nid_request = true;
+                       break;
                case 'i':
                        rc = lustre_lnet_parse_interfaces(optarg, &nw_descr);
                        if (rc != 0) {
@@ -2605,6 +2639,14 @@ static int jt_del_ni(int argc, char **argv)
                }
        }
 
+       if (nid_request) {
+               if (net_id) {
+                       fprintf(stdout, "--net is invalid with --nid option\n");
+                       return -EINVAL;
+               }
+               net_id = libcfs_net2str(nw_descr.nw_id);
+       }
+
        rc = yaml_lnet_config_ni(net_id, NULL, &nw_descr, NULL, -1, NULL,
                                 LNET_GENL_VERSION, 0);
        if (rc <= 0) {

diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh
index 78cc861..366099b 100755 (executable)
--- a/lustre/tests/sanity-lnet.sh
+++ b/lustre/tests/sanity-lnet.sh
@@ -1486,6 +1486,33 @@ test_109() {
 }
 run_test 109 "Add NI using a network interface alias (LU-16859)"
 
+test_110() {
+       [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE"
+
+       cleanup_netns || error "Failed to cleanup netns before test execution"
+       cleanup_lnet || error "Failed to unload modules before test execution"
+
+       configure_dlc || error "Failed to configure DLC rc = $?"
+
+       LOCAL_ADDR_LIST=$(local_addr_list)
+       set -- $LOCAL_ADDR_LIST
+       do_lnetctl net add --nid $2@tcp ||
+               error "Failed to add net tcp for IP $2@tcp"
+
+       $LNETCTL net show --net tcp | grep -q "nid: $2@tcp" ||
+               error "Failed to configure $2@tcp"
+
+       do_lnetctl net del --nid $2@tcp ||
+               error "Failed to del net tcp for IP $2@tcp"
+
+       $LNETCTL net show | grep -q "nid: $2@tcp"&&
+               error "$2@tcp should have been deleted"
+
+       cleanup_lnet
+       setup_netns
+}
+run_test 110 "Add NI using a specific TCP / IP address"
+
 test_200() {
        [[ ${NETTYPE} == tcp* ]] ||
                skip "Need tcp NETTYPE"