LU-10003 lnet: migrate lnet setup and tear down to Netlink

Migrate the LNet setup and tear down functionality from ioctls to
Netlink. This change now means lnet_ioctl() is no longer needed but
we will keep it for now to support older tools. The work here will
be used in a follow on patch to tell lnet to setup large NIDs by
default for testing.

Test-Parameters: trivial testlist=sanity-lnet
Change-Id: Id69810e114818d423102d6e85ff93529f04c337f
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54359
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index ad33e40..4ec0387 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -500,6 +500,9 @@ lnet_rspt_free(struct lnet_rsp_tracker *rspt, int cpt)
        lnet_net_unlock(cpt);
 }
 
+int lnet_configure(void *arg);
+int lnet_unconfigure(void);
+
 void lnet_ni_free(struct lnet_ni *ni);
 void lnet_net_free(struct lnet_net *net);
 

diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h
index 8b51a99..f3afed9 100644 (file)
--- a/lnet/include/uapi/linux/lnet/lnet-dlc.h
+++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h
@@ -56,6 +56,7 @@
  *
  * @LNET_CMD_UNSPEC:           unspecified command to catch errors
  *
+ * @LNET_CMD_CONFIGURE:                command to setup LNet stack
  * @LNET_CMD_NETS:             command to manage the LNet networks
  * @LNET_CMD_ROUTES:           command to manage LNet routes
  * @LNET_CMD_PING:             command to send pings to LNet connections

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 1fe475f..911bdd9 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -4791,6 +4791,38 @@ report_discover_err:
 }
 EXPORT_SYMBOL(LNetCtl);
 
+static int lnet_net_conf_cmd(struct sk_buff *skb, struct genl_info *info)
+{
+       int rc = 0;
+
+       if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
+               /* NLM_F_EXCL means ignore module parameters */
+               if (info->nlhdr->nlmsg_flags & NLM_F_EXCL)
+                       the_lnet.ln_nis_from_mod_params = true;
+
+               rc = lnet_configure(NULL);
+               switch (rc) {
+               case -ENETDOWN:
+                       GENL_SET_ERR_MSG(info,
+                                        "Network is down");
+                       break;
+               case -ENODEV:
+                       GENL_SET_ERR_MSG(info,
+                                        "LNET is currently not loaded");
+                       break;
+               case -EBUSY:
+                       GENL_SET_ERR_MSG(info, "LNET busy");
+                       break;
+               default:
+                       break;
+               }
+       } else {
+               rc = lnet_unconfigure();
+       }
+
+       return rc;
+};
+
 struct lnet_nid_cpt {
        struct lnet_nid lnc_nid;
        unsigned int lnc_cpt;
@@ -8825,6 +8857,11 @@ static const struct genl_multicast_group lnet_mcast_grps[] = {
 
 static const struct genl_ops lnet_genl_ops[] = {
        {
+               .cmd            = LNET_CMD_CONFIGURE,
+               .flags          = GENL_ADMIN_PERM,
+               .doit           = lnet_net_conf_cmd,
+       },
+       {
                .cmd            = LNET_CMD_NETS,
                .flags          = GENL_ADMIN_PERM,
 #ifdef HAVE_NETLINK_CALLBACK_START

diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c
index ad63090..ead3cf5 100644 (file)
--- a/lnet/lnet/module.c
+++ b/lnet/lnet/module.c
@@ -41,7 +41,7 @@ MODULE_PARM_DESC(config_on_load, "configure network at module load");
 
 static DEFINE_MUTEX(lnet_config_mutex);
 
-static int lnet_configure(void *arg)
+int lnet_configure(void *arg)
 {
        /* 'arg' only there so I can be passed to cfs_create_thread() */
        int    rc = 0;
@@ -68,7 +68,7 @@ out:
        return rc;
 }
 
-static int lnet_unconfigure(void)
+int lnet_unconfigure(void)
 {
        int refcount;
 

diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index e2c0afe..958b5cb 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -407,6 +407,81 @@ int lustre_lnet_config_ni_system(bool up, bool load_ni_from_mod,
        return rc;
 }
 
+int yaml_lnet_configure(int flags, const char **msg)
+{
+       char *err = "lnet configure: ";
+       yaml_parser_t setup, reply;
+       char *config = "net:\n";
+       yaml_document_t results;
+       yaml_emitter_t request;
+       struct nl_sock *sk;
+       int rc;
+
+       /* Initialize configuration parser */
+       rc = yaml_parser_initialize(&setup);
+       if (rc == 0) {
+               yaml_parser_log_error(&setup, stderr, err);
+               yaml_parser_delete(&setup);
+               return -EOPNOTSUPP;
+       }
+
+       yaml_parser_set_input_string(&setup, (unsigned char *)config,
+                                    strlen(config));
+       rc = yaml_parser_load(&setup, &results);
+       if (rc == 0) {
+               yaml_parser_log_error(&setup, stderr, err);
+               yaml_parser_delete(&setup);
+               return -EOPNOTSUPP;
+       }
+       yaml_parser_delete(&setup);
+
+       /* Create Netlink emitter to send request to kernel */
+       sk = nl_socket_alloc();
+       if (!sk) {
+               yaml_document_delete(&results);
+               return -EOPNOTSUPP;
+       }
+
+       /* Setup parser to recieve Netlink packets */
+       rc = yaml_parser_initialize(&reply);
+       if (rc == 0) {
+               yaml_document_delete(&results);
+               nl_socket_free(sk);
+               return -EOPNOTSUPP;
+       }
+
+       rc = yaml_parser_set_input_netlink(&reply, sk, false);
+       if (rc == 0)
+               goto free_reply;
+
+       yaml_emitter_initialize(&request);
+       rc = yaml_emitter_set_output_netlink(&request, sk, LNET_GENL_NAME,
+                                            LNET_GENL_VERSION,
+                                            LNET_CMD_CONFIGURE, flags);
+       if (rc == 1) /* 1 is success */
+               rc = yaml_emitter_dump(&request, &results);
+       if (rc == 0) {
+               yaml_emitter_log_error(&request, stderr);
+               rc = -EINVAL;
+       } else {
+               yaml_document_t errmsg;
+
+               rc = yaml_parser_load(&reply, &errmsg);
+       }
+       yaml_emitter_delete(&request);
+free_reply:
+       if (rc == 0) {
+               *msg = yaml_parser_get_reader_error(&reply);
+               rc = errno;
+       }
+
+       yaml_parser_delete(&reply);
+       yaml_document_delete(&results);
+       nl_socket_free(sk);
+
+       return rc == 1 ? 0 : rc;
+}
+
 static int dispatch_peer_ni_cmd(__u32 cmd, struct lnet_ioctl_peer_cfg *data,
                                char *err_str, char *cmd_str)
 {

diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h
index 8a482a7..47d5903 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig.h
+++ b/lnet/utils/lnetconfig/liblnetconfig.h
@@ -793,6 +793,20 @@ int lustre_lnet_parse_nid_range(struct nid_node *head, char *nidstr,
 void lustre_lnet_free_list(struct nid_node *head);
 
 /**
+ * yaml_lnet_configure
+ *
+ *   Shared code for LNet setup and teardown.
+ *
+ *   flags     - Netlink flags that control setup behavior
+ *               NLM_F_CREATE  - initializes LNet stack
+ *               NLM_F_EXCL    - ignore module params
+ *               0             - tears down LNet stack
+ *
+ *   msg         error msg string returned.
+ */
+int yaml_lnet_configure(int flags, const char **msg);
+
+/**
  * yaml_emitter_set_output_netlink
  *
  *   Special handling to integrate LNet handling into libyaml.

diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c
index 299b367..4ee44b9 100644 (file)
--- a/lnet/utils/lnetctl.c
+++ b/lnet/utils/lnetctl.c
@@ -899,6 +899,11 @@ static void yaml_lnet_print_error(int op, char *cmd, const char *errstr)
        if (rc == 0)
                goto emitter_error;
 
+       if (strcmp(cmd, "lnet") == 0) {
+               flag = "configure";
+               goto skip_op;
+       }
+
        switch (op) {
        case NLM_F_CREATE:
                flag = "add";
@@ -917,7 +922,7 @@ static void yaml_lnet_print_error(int op, char *cmd, const char *errstr)
                flag = "show";
                break;
        }
-
+skip_op:
        yaml_scalar_event_initialize(&event, NULL,
                                     (yaml_char_t *)YAML_STR_TAG,
                                     (yaml_char_t *)flag,
@@ -1221,6 +1226,8 @@ static int jt_config_lnet(int argc, char **argv)
 {
        struct cYAML *err_rc = NULL;
        bool load_mod_params = false;
+       int flags = NLM_F_CREATE;
+       const char *msg = NULL;
        int rc, opt;
 
        const char *const short_options = "a";
@@ -1244,6 +1251,21 @@ static int jt_config_lnet(int argc, char **argv)
                }
        }
 
+       if (!load_mod_params)
+               flags |= NLM_F_EXCL;
+
+       rc = yaml_lnet_configure(flags, &msg);
+       if (rc != -EOPNOTSUPP) {
+               if (rc < 0) {
+                       char errstr[256];
+
+                       snprintf(errstr, sizeof(errstr),
+                                "LNet configure error: %s", msg);
+                       yaml_lnet_print_error(flags, "lnet", errstr);
+               }
+               return rc;
+       }
+
        rc = lustre_lnet_config_ni_system(LNET_CONFIGURE, load_mod_params,
                                          -1, &err_rc);
 
@@ -1258,12 +1280,25 @@ static int jt_config_lnet(int argc, char **argv)
 static int jt_unconfig_lnet(int argc, char **argv)
 {
        struct cYAML *err_rc = NULL;
+       const char *msg = NULL;
        int rc;
 
        rc = check_cmd(lnet_cmds, "lnet", "unconfigure", 0, argc, argv);
        if (rc)
                return rc;
 
+       rc = yaml_lnet_configure(0, &msg);
+       if (rc != -EOPNOTSUPP) {
+               if (rc < 0) {
+                       char errstr[256];
+
+                       snprintf(errstr, sizeof(errstr),
+                                "LNet configure error: %s", msg);
+                       yaml_lnet_print_error(0, "lnet", errstr);
+               }
+               return rc;
+       }
+
        rc = lustre_lnet_config_ni_system(LNET_UNCONFIGURE, 0, -1, &err_rc);
 
        if (rc != LUSTRE_CFG_RC_NO_ERR)

diff --git a/lustre/utils/portals.c b/lustre/utils/portals.c
index e9b7129..0e928bc 100644 (file)
--- a/lustre/utils/portals.c
+++ b/lustre/utils/portals.c
@@ -304,6 +304,7 @@ int jt_ptl_network(int argc, char **argv)
 {
        struct libcfs_ioctl_data data;
        __u32 net = LNET_NET_ANY;
+       const char *msg = NULL;
        int rc;
 
        if (argc != 2) {
@@ -312,9 +313,26 @@ int jt_ptl_network(int argc, char **argv)
        }
 
        if (!strcmp(argv[1], "unconfigure") || !strcmp(argv[1], "down")) {
+               rc = yaml_lnet_configure(0, &msg);
+               if (rc != -EOPNOTSUPP) {
+                       switch (rc) {
+                       case 0:
+                               printf("LNET ready to unload\n");
+                               break;
+                       case -ENODEV:
+                       case -EBUSY:
+                               printf("%s\n", msg);
+                               break;
+                       default:
+                               printf("LNET unconfigure error %u: %s\n",
+                                      -rc, msg);
+                               break;
+                       }
+                       return rc;
+               }
+
                LIBCFS_IOC_INIT(data);
                rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_UNCONFIGURE, &data);
-
                if (rc == 0) {
                        printf("LNET ready to unload\n");
                        return 0;
@@ -332,6 +350,20 @@ int jt_ptl_network(int argc, char **argv)
                                errno, strerror(errno));
                return -1;
        } else if (!strcmp(argv[1], "configure") || !strcmp(argv[1], "up")) {
+               rc = yaml_lnet_configure(NLM_F_CREATE, &msg);
+               if (rc != -EOPNOTSUPP) {
+                       switch (rc) {
+                       case 0:
+                               printf("LNET configured\n");
+                               break;
+                       default:
+                               fprintf(stderr, "LNET configure error %u: %s\n",
+                                       -rc, msg);
+                               break;
+                       }
+                       return rc;
+               }
+
                LIBCFS_IOC_INIT(data);
                rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_CONFIGURE, &data);