Whamcloud - gitweb
LU-16822 lnet: always initialize IPv6 at start up 18/51818/6
authorJames Simmons <jsimmons@infradead.org>
Mon, 22 Apr 2024 14:08:11 +0000 (10:08 -0400)
committerOleg Drokin <green@whamcloud.com>
Tue, 30 Apr 2024 06:45:45 +0000 (06:45 +0000)
Currently lnet_inet_enumerate() has a bool parameter that enables
collecting IPv6 addresses for selection which is optional. This
patch changes the behavior to always collect proper IPv6 addresses
and now the bool flag means prefer the IPv6 over any IPv4 addresses.
Update the user land applications lctl and lnetctl to send a flag
to select IPv6 or IPv4 at initialization of the LNet stack. This
is useful for IPv6 and other large NID type testing.

Test-Parameters: trivial testlist=sanity-lnet
Change-Id: Ib3f38de15b1295ec1f8e8607dbd971583541f06c
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51818
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
lnet/include/lnet/lib-types.h
lnet/klnds/socklnd/socklnd.c
lnet/lnet/api-ni.c
lnet/lnet/config.c
lnet/lnet/lib-socket.c
lnet/utils/lnetctl.c
lustre/utils/portals.c

index 210de84..b5de29e 100644 (file)
@@ -1929,6 +1929,9 @@ struct lnet {
         */
        bool                            ln_nis_from_mod_params;
 
+       /* Switch to test large NIDs */
+       bool                            ln_nis_use_large_nids;
+
        /*
         * completion for the monitor thread. The monitor thread takes care of
         * checking routes, timedout messages and resending messages.
index d187969..ecea6a9 100644 (file)
@@ -2592,7 +2592,8 @@ ksocknal_startup(struct lnet_ni *ni)
 
        ksocknal_tunables_setup(ni);
 
-       rc = lnet_inet_enumerate(&ifaces, ni->ni_net_ns, true);
+       rc = lnet_inet_enumerate(&ifaces, ni->ni_net_ns,
+                                ni->ni_net->net_tunables.lct_version);
        if (rc < 0)
                goto out_net;
 
index 973b615..ed1d157 100644 (file)
@@ -4780,6 +4780,9 @@ static int lnet_net_conf_cmd(struct sk_buff *skb, struct genl_info *info)
                if (info->nlhdr->nlmsg_flags & NLM_F_EXCL)
                        the_lnet.ln_nis_from_mod_params = true;
 
+               if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE)
+                       the_lnet.ln_nis_use_large_nids = true;
+
                rc = lnet_configure(NULL);
                switch (rc) {
                case -ENETDOWN:
index 85ba572..01c61b7 100644 (file)
 #define DEBUG_SUBSYSTEM S_LNET
 
 #include <linux/ctype.h>
-#include <linux/inetdevice.h>
 #include <linux/nsproxy.h>
 #include <linux/ethtool.h>
 #include <net/net_namespace.h>
 #include <lnet/lib-lnet.h>
-#include <net/addrconf.h>
 
 /* tmp struct for parsing routes */
 struct lnet_text_buf {
@@ -352,6 +350,7 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
        net->net_tunables.lct_max_tx_credits = -1;
        net->net_tunables.lct_peer_tx_credits = -1;
        net->net_tunables.lct_peer_rtr_credits = -1;
+       net->net_tunables.lct_version = 0;
 
        if (net_list)
                list_add_tail(&net->net_list, net_list);
@@ -686,6 +685,9 @@ lnet_parse_networks(struct list_head *netlist, const char *networks)
                if (IS_ERR_OR_NULL(net))
                        goto failed;
 
+               if (the_lnet.ln_nis_use_large_nids)
+                       net->net_tunables.lct_version = 1;
+
                if (!nistr) {
                        /*
                         * No interface list was specified, allocate a
@@ -1512,134 +1514,6 @@ int lnet_get_link_status(struct net_device *dev)
 }
 EXPORT_SYMBOL(lnet_get_link_status);
 
-int lnet_inet_enumerate(struct lnet_inetdev **dev_list, struct net *ns, bool v6)
-{
-       struct lnet_inetdev *ifaces = NULL;
-       struct net_device *dev;
-       int nalloc = 0;
-       int nip = 0;
-       DECLARE_CONST_IN_IFADDR(ifa);
-
-       rtnl_lock();
-       for_each_netdev(ns, dev) {
-               int flags = dev_get_flags(dev);
-               struct in_device *in_dev;
-#if IS_ENABLED(CONFIG_IPV6)
-               struct inet6_dev *in6_dev;
-               const struct inet6_ifaddr *ifa6;
-#endif
-               int node_id;
-               int cpt;
-
-               if (flags & IFF_LOOPBACK) /* skip the loopback IF */
-                       continue;
-
-               if (!(flags & IFF_UP)) {
-                       CWARN("lnet: Ignoring interface %s: it's down\n",
-                             dev->name);
-                       continue;
-               }
-
-               node_id = dev_to_node(&dev->dev);
-               cpt = cfs_cpt_of_node(lnet_cpt_table(), node_id);
-
-               in_dev = __in_dev_get_rtnl(dev);
-               if (!in_dev) {
-                       if (!v6)
-                               CWARN("lnet: Interface %s has no IPv4 status.\n",
-                                     dev->name);
-                       goto try_v6;
-               }
-
-               in_dev_for_each_ifa_rtnl(ifa, in_dev) {
-                       if (nip >= nalloc) {
-                               struct lnet_inetdev *tmp;
-
-                               nalloc += LNET_INTERFACES_NUM;
-                               tmp = krealloc(ifaces, nalloc * sizeof(*tmp),
-                                              GFP_KERNEL);
-                               if (!tmp) {
-                                       kfree(ifaces);
-                                       ifaces = NULL;
-                                       nip = -ENOMEM;
-                                       goto unlock_rtnl;
-                               }
-                               ifaces = tmp;
-                       }
-
-                       ifaces[nip].li_cpt = cpt;
-                       ifaces[nip].li_iff_master = !!(flags & IFF_MASTER);
-                       ifaces[nip].li_size = sizeof(ifa->ifa_local);
-                       ifaces[nip].li_index = dev->ifindex;
-                       ifaces[nip].li_ipaddr = ifa->ifa_local;
-                       ifaces[nip].li_netmask = ntohl(ifa->ifa_mask);
-                       strscpy(ifaces[nip].li_name, ifa->ifa_label,
-                               sizeof(ifaces[nip].li_name));
-                       nip++;
-               }
-               endfor_ifa(in_dev);
-
-       try_v6:
-               if (!v6)
-                       continue;
-#if IS_ENABLED(CONFIG_IPV6)
-               in6_dev = __in6_dev_get(dev);
-               if (!in6_dev) {
-                       if (!in_dev)
-                               CWARN("lnet: Interface %s has no IP status.\n",
-                                     dev->name);
-                       continue;
-               }
-
-               list_for_each_entry_rcu(ifa6, &in6_dev->addr_list, if_list) {
-                       if (ifa6->flags & IFA_F_TEMPORARY)
-                               continue;
-                       if (nip >= nalloc) {
-                               struct lnet_inetdev *tmp;
-
-                               nalloc += LNET_INTERFACES_NUM;
-                               tmp = krealloc(ifaces, nalloc * sizeof(*tmp),
-                                              GFP_KERNEL);
-                               if (!tmp) {
-                                       kfree(ifaces);
-                                       ifaces = NULL;
-                                       nip = -ENOMEM;
-                                       goto unlock_rtnl;
-                               }
-                               ifaces = tmp;
-                       }
-
-                       ifaces[nip].li_cpt = cpt;
-                       ifaces[nip].li_iff_master = !!(flags & IFF_MASTER);
-                       ifaces[nip].li_size = sizeof(struct in6_addr);
-                       ifaces[nip].li_index = dev->ifindex;
-                       memcpy(ifaces[nip].li_ipv6addr,
-                              &ifa6->addr, sizeof(struct in6_addr));
-                       strscpy(ifaces[nip].li_name, dev->name,
-                               sizeof(ifaces[nip].li_name));
-                       nip++;
-                       /* As different IPv6 addresses don't have unique
-                        * labels, it is safest just to use the first
-                        * and ignore the rest.
-                        */
-                       break;
-               }
-#endif /* IS_ENABLED(CONFIG_IPV6) */
-
-       }
-unlock_rtnl:
-       rtnl_unlock();
-
-       if (nip == 0) {
-               CERROR("lnet: Can't find any usable interfaces, rc = -ENOENT\n");
-               nip = -ENOENT;
-       }
-
-       *dev_list = ifaces;
-       return nip;
-}
-EXPORT_SYMBOL(lnet_inet_enumerate);
-
 int lnet_inet_select(struct lnet_ni *ni,
                     struct lnet_inetdev *ifaces,
                      int num_ifaces)
@@ -1694,16 +1568,17 @@ int
 lnet_parse_ip2nets(const char **networksp, const char *ip2nets)
 {
        struct lnet_inetdev *ifaces = NULL;
-       __u32     *ipaddrs = NULL;
+       u32 *ipaddrs = NULL;
        int nip;
-       int        rc;
+       int rc;
        int i;
 
        if (current->nsproxy && current->nsproxy->net_ns)
                nip = lnet_inet_enumerate(&ifaces, current->nsproxy->net_ns,
-                                         false);
+                                         the_lnet.ln_nis_use_large_nids);
        else
-               nip = lnet_inet_enumerate(&ifaces, &init_net, false);
+               nip = lnet_inet_enumerate(&ifaces, &init_net,
+                                         the_lnet.ln_nis_use_large_nids);
        if (nip < 0) {
                if (nip != -ENOENT) {
                        LCONSOLE_ERROR_MSG(0x117,
index cf533b3..e436289 100644 (file)
@@ -423,3 +423,180 @@ lnet_sock_connect(int interface, int local_port,
        sock_release(sock);
        return ERR_PTR(rc);
 }
+
+static int lnet_inet4_enumerate(struct net_device *dev, int flags,
+                               int *nalloc, int nip, int cpt,
+                               struct lnet_inetdev **dev_list)
+{
+       struct lnet_inetdev *ifaces = *dev_list;
+       struct in_device *in_dev;
+       DECLARE_CONST_IN_IFADDR(ifa);
+
+       in_dev = __in_dev_get_rtnl(dev);
+       if (!in_dev) {
+               CWARN("lnet: Interface %s has no IPv4 status.\n",
+                     dev->name);
+               return nip;
+       }
+
+       in_dev_for_each_ifa_rtnl(ifa, in_dev) {
+               if (nip >= *nalloc) {
+                       struct lnet_inetdev *tmp;
+
+                       *nalloc += LNET_INTERFACES_NUM;
+                       tmp = krealloc(ifaces, *nalloc * sizeof(*tmp),
+                                      GFP_KERNEL);
+                       if (!tmp) {
+                               kfree(ifaces);
+                               ifaces = NULL;
+                               return -ENOMEM;
+                       }
+                       ifaces = tmp;
+               }
+
+               ifaces[nip].li_cpt = cpt;
+               ifaces[nip].li_iff_master = !!(flags & IFF_MASTER);
+               ifaces[nip].li_size = sizeof(ifa->ifa_local);
+               ifaces[nip].li_index = dev->ifindex;
+               ifaces[nip].li_ipaddr = ifa->ifa_local;
+               ifaces[nip].li_netmask = ntohl(ifa->ifa_mask);
+               strscpy(ifaces[nip].li_name, ifa->ifa_label,
+                      sizeof(ifaces[nip].li_name));
+               nip++;
+       }
+       endfor_ifa(in_dev);
+
+       *dev_list = ifaces;
+
+       return nip;
+}
+
+static int lnet_inet6_enumerate(struct net_device *dev, int flags,
+                               int *nalloc, int nip, int cpt,
+                               struct lnet_inetdev **dev_list)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+       struct lnet_inetdev *ifaces = *dev_list;
+       const struct inet6_ifaddr *ifa6;
+       struct inet6_dev *in6_dev;
+
+       in6_dev = __in6_dev_get(dev);
+       if (!in6_dev) {
+               CWARN("lnet: Interface %s has no IPv6 status.\n",
+                     dev->name);
+               return nip;
+       }
+
+       list_for_each_entry_rcu(ifa6, &in6_dev->addr_list, if_list) {
+               if (ifa6->flags & IFA_F_TEMPORARY)
+                       continue;
+
+               if (ipv6_addr_type(&ifa6->addr) & IPV6_ADDR_LINKLOCAL)
+                       continue;
+
+               if (nip >= *nalloc) {
+                       struct lnet_inetdev *tmp;
+
+                       *nalloc += LNET_INTERFACES_NUM;
+                       tmp = krealloc(ifaces, *nalloc * sizeof(*tmp),
+                                      GFP_KERNEL);
+                       if (!tmp) {
+                               kfree(ifaces);
+                               ifaces = NULL;
+                               return -ENOMEM;
+                       }
+                       ifaces = tmp;
+               }
+
+               ifaces[nip].li_cpt = cpt;
+               ifaces[nip].li_iff_master = !!(flags & IFF_MASTER);
+               ifaces[nip].li_size = sizeof(struct in6_addr);
+               ifaces[nip].li_index = dev->ifindex;
+               memcpy(ifaces[nip].li_ipv6addr,
+                      &ifa6->addr, sizeof(struct in6_addr));
+               strscpy(ifaces[nip].li_name, dev->name,
+                       sizeof(ifaces[nip].li_name));
+               nip++;
+               /* As different IPv6 addresses don't have unique
+                * labels, it is safest just to use the first
+                * and ignore the rest.
+                */
+               break;
+       }
+
+       *dev_list = ifaces;
+#endif /* IS_ENABLED(CONFIG_IPV6) */
+       return nip;
+}
+
+int lnet_inet_enumerate(struct lnet_inetdev **dev_list, struct net *ns,
+                       bool v6_first)
+{
+       struct lnet_inetdev *ifaces = NULL;
+       struct net_device *dev;
+       int nalloc = 0;
+       int nip = 0;
+
+       rtnl_lock();
+       for_each_netdev(ns, dev) {
+               int flags = dev_get_flags(dev);
+               int node_id, cpt;
+               int count;
+
+               if (flags & IFF_LOOPBACK) /* skip the loopback IF */
+                       continue;
+
+               if (!(flags & IFF_UP)) {
+                       CWARN("lnet: Ignoring interface %s: it's down\n",
+                             dev->name);
+                       continue;
+               }
+
+               node_id = dev_to_node(&dev->dev);
+               cpt = cfs_cpt_of_node(lnet_cpt_table(), node_id);
+
+               if (v6_first) {
+                       count = lnet_inet6_enumerate(dev, flags, &nalloc, nip,
+                                                    cpt, &ifaces);
+                       if (count < 0)
+                               CWARN("lnet: No IPv6 addresses for interface %s.\n",
+                                     dev->name);
+                       else
+                               nip = count;
+
+                       count = lnet_inet4_enumerate(dev, flags, &nalloc, nip,
+                                                    cpt, &ifaces);
+                       if (count < 0)
+                               CWARN("lnet: No IPv4 addresses for interface %s.\n",
+                                     dev->name);
+                       else
+                               nip = count;
+               } else {
+                       count = lnet_inet4_enumerate(dev, flags, &nalloc, nip,
+                                                    cpt, &ifaces);
+                       if (count < 0)
+                               CWARN("lnet: No IPv4 addresses for interface %s.\n",
+                                     dev->name);
+                       else
+                               nip = count;
+
+                       count = lnet_inet6_enumerate(dev, flags, &nalloc, nip,
+                                                    cpt, &ifaces);
+                       if (count < 0)
+                               CWARN("lnet: No IPv6 addresses for interface %s.\n",
+                                     dev->name);
+                       else
+                               nip = count;
+               }
+       }
+       rtnl_unlock();
+
+       if (nip == 0) {
+               CERROR("lnet: Can't find any usable interfaces, rc = -ENOENT\n");
+               nip = -ENOENT;
+       }
+
+       *dev_list = ifaces;
+       return nip;
+}
+EXPORT_SYMBOL(lnet_inet_enumerate);
index 25c4b73..02ba1ba 100644 (file)
@@ -87,7 +87,7 @@ static int jt_calc_cpt_of_nid(int argc, char **argv);
 static int jt_show_peer_debug_info(int argc, char **argv);
 
 command_t cmd_list[] = {
-       {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all]"},
+       {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all|--large]"},
        {"route", jt_route, 0, "route {add | del | show | help}"},
        {"net", jt_net, 0, "net {add | del | show | set | help}"},
        {"routing", jt_routing, 0, "routing {show | help}"},
@@ -117,7 +117,8 @@ command_t cmd_list[] = {
 
 command_t lnet_cmds[] = {
        {"configure", jt_config_lnet, 0, "configure lnet\n"
-        "\t--all: load NI configuration from module parameters\n"},
+        "\t--all: load NI configuration from module parameters\n"
+        "\t--large: start LNet with large NIDs\n"},
        {"unconfigure", jt_unconfig_lnet, 0, "unconfigure lnet\n"},
        { 0, 0, 0, NULL }
 };
@@ -1232,10 +1233,10 @@ static int jt_config_lnet(int argc, char **argv)
        int flags = NLM_F_CREATE;
        const char *msg = NULL;
        int rc, opt;
-
-       const char *const short_options = "a";
+       const char *const short_options = "al";
        static const struct option long_options[] = {
-               { .name = "all",  .has_arg = no_argument, .val = 'a' },
+               { .name = "all",        .has_arg = no_argument, .val = 'a' },
+               { .name = "large",      .has_arg = no_argument, .val = 'l' },
                { .name = NULL }
        };
 
@@ -1249,6 +1250,9 @@ static int jt_config_lnet(int argc, char **argv)
                case 'a':
                        load_mod_params = true;
                        break;
+               case 'l':
+                       flags |= NLM_F_REPLACE;
+                       break;
                default:
                        return 0;
                }
index 0e928bc..dce10e6 100644 (file)
@@ -307,8 +307,8 @@ int jt_ptl_network(int argc, char **argv)
        const char *msg = NULL;
        int rc;
 
-       if (argc != 2) {
-               fprintf(stderr, "usage: %s <net>|up|down\n", argv[0]);
+       if (argc > 3) {
+               fprintf(stderr, "usage: %s <net>|up|down [-l]\n", argv[0]);
                return -1;
        }
 
@@ -325,7 +325,7 @@ int jt_ptl_network(int argc, char **argv)
                                break;
                        default:
                                printf("LNET unconfigure error %u: %s\n",
-                                      -rc, msg);
+                                      -rc, msg ? msg : strerror(-rc));
                                break;
                        }
                        return rc;
@@ -350,7 +350,12 @@ int jt_ptl_network(int argc, char **argv)
                                errno, strerror(errno));
                return -1;
        } else if (!strcmp(argv[1], "configure") || !strcmp(argv[1], "up")) {
-               rc = yaml_lnet_configure(NLM_F_CREATE, &msg);
+               int flags = NLM_F_CREATE;
+
+               if (argc == 3 && argv[2] && !strcmp(argv[2], "-l"))
+                       flags |= NLM_F_REPLACE;
+
+               rc = yaml_lnet_configure(flags, &msg);
                if (rc != -EOPNOTSUPP) {
                        switch (rc) {
                        case 0: