From 6526fdb007a5110791f0b49ef61d2518d9b14e44 Mon Sep 17 00:00:00 2001 From: James Simmons Date: Tue, 12 Mar 2024 09:24:19 -0400 Subject: [PATCH] LU-10003 lnet: migrate lnet setup and tear down to Netlink Migrate the LNet setup and tear down functionality from ioctls to Netlink. This change now means lnet_ioctl() is no longer needed but we will keep it for now to support older tools. The work here will be used in a follow on patch to tell lnet to setup large NIDs by default for testing. Test-Parameters: trivial testlist=sanity-lnet Change-Id: Id69810e114818d423102d6e85ff93529f04c337f Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54359 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Chris Horn Reviewed-by: Serguei Smirnov Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-lnet.h | 3 ++ lnet/include/uapi/linux/lnet/lnet-dlc.h | 1 + lnet/lnet/api-ni.c | 37 ++++++++++++++++ lnet/lnet/module.c | 4 +- lnet/utils/lnetconfig/liblnetconfig.c | 75 +++++++++++++++++++++++++++++++++ lnet/utils/lnetconfig/liblnetconfig.h | 14 ++++++ lnet/utils/lnetctl.c | 37 +++++++++++++++- lustre/utils/portals.c | 34 ++++++++++++++- 8 files changed, 201 insertions(+), 4 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index ad33e40..4ec0387 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -500,6 +500,9 @@ lnet_rspt_free(struct lnet_rsp_tracker *rspt, int cpt) lnet_net_unlock(cpt); } +int lnet_configure(void *arg); +int lnet_unconfigure(void); + void lnet_ni_free(struct lnet_ni *ni); void lnet_net_free(struct lnet_net *net); diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h index 8b51a99..f3afed9 100644 --- a/lnet/include/uapi/linux/lnet/lnet-dlc.h +++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h @@ -56,6 +56,7 @@ * * @LNET_CMD_UNSPEC: unspecified command to catch errors * + * @LNET_CMD_CONFIGURE: command to setup LNet stack * @LNET_CMD_NETS: command to manage the LNet networks * @LNET_CMD_ROUTES: command to manage LNet routes * @LNET_CMD_PING: command to send pings to LNet connections diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 1fe475f..911bdd9 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -4791,6 +4791,38 @@ report_discover_err: } EXPORT_SYMBOL(LNetCtl); +static int lnet_net_conf_cmd(struct sk_buff *skb, struct genl_info *info) +{ + int rc = 0; + + if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) { + /* NLM_F_EXCL means ignore module parameters */ + if (info->nlhdr->nlmsg_flags & NLM_F_EXCL) + the_lnet.ln_nis_from_mod_params = true; + + rc = lnet_configure(NULL); + switch (rc) { + case -ENETDOWN: + GENL_SET_ERR_MSG(info, + "Network is down"); + break; + case -ENODEV: + GENL_SET_ERR_MSG(info, + "LNET is currently not loaded"); + break; + case -EBUSY: + GENL_SET_ERR_MSG(info, "LNET busy"); + break; + default: + break; + } + } else { + rc = lnet_unconfigure(); + } + + return rc; +}; + struct lnet_nid_cpt { struct lnet_nid lnc_nid; unsigned int lnc_cpt; @@ -8825,6 +8857,11 @@ static const struct genl_multicast_group lnet_mcast_grps[] = { static const struct genl_ops lnet_genl_ops[] = { { + .cmd = LNET_CMD_CONFIGURE, + .flags = GENL_ADMIN_PERM, + .doit = lnet_net_conf_cmd, + }, + { .cmd = LNET_CMD_NETS, .flags = GENL_ADMIN_PERM, #ifdef HAVE_NETLINK_CALLBACK_START diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index ad63090..ead3cf5 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -41,7 +41,7 @@ MODULE_PARM_DESC(config_on_load, "configure network at module load"); static DEFINE_MUTEX(lnet_config_mutex); -static int lnet_configure(void *arg) +int lnet_configure(void *arg) { /* 'arg' only there so I can be passed to cfs_create_thread() */ int rc = 0; @@ -68,7 +68,7 @@ out: return rc; } -static int lnet_unconfigure(void) +int lnet_unconfigure(void) { int refcount; diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index e2c0afe..958b5cb 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -407,6 +407,81 @@ int lustre_lnet_config_ni_system(bool up, bool load_ni_from_mod, return rc; } +int yaml_lnet_configure(int flags, const char **msg) +{ + char *err = "lnet configure: "; + yaml_parser_t setup, reply; + char *config = "net:\n"; + yaml_document_t results; + yaml_emitter_t request; + struct nl_sock *sk; + int rc; + + /* Initialize configuration parser */ + rc = yaml_parser_initialize(&setup); + if (rc == 0) { + yaml_parser_log_error(&setup, stderr, err); + yaml_parser_delete(&setup); + return -EOPNOTSUPP; + } + + yaml_parser_set_input_string(&setup, (unsigned char *)config, + strlen(config)); + rc = yaml_parser_load(&setup, &results); + if (rc == 0) { + yaml_parser_log_error(&setup, stderr, err); + yaml_parser_delete(&setup); + return -EOPNOTSUPP; + } + yaml_parser_delete(&setup); + + /* Create Netlink emitter to send request to kernel */ + sk = nl_socket_alloc(); + if (!sk) { + yaml_document_delete(&results); + return -EOPNOTSUPP; + } + + /* Setup parser to recieve Netlink packets */ + rc = yaml_parser_initialize(&reply); + if (rc == 0) { + yaml_document_delete(&results); + nl_socket_free(sk); + return -EOPNOTSUPP; + } + + rc = yaml_parser_set_input_netlink(&reply, sk, false); + if (rc == 0) + goto free_reply; + + yaml_emitter_initialize(&request); + rc = yaml_emitter_set_output_netlink(&request, sk, LNET_GENL_NAME, + LNET_GENL_VERSION, + LNET_CMD_CONFIGURE, flags); + if (rc == 1) /* 1 is success */ + rc = yaml_emitter_dump(&request, &results); + if (rc == 0) { + yaml_emitter_log_error(&request, stderr); + rc = -EINVAL; + } else { + yaml_document_t errmsg; + + rc = yaml_parser_load(&reply, &errmsg); + } + yaml_emitter_delete(&request); +free_reply: + if (rc == 0) { + *msg = yaml_parser_get_reader_error(&reply); + rc = errno; + } + + yaml_parser_delete(&reply); + yaml_document_delete(&results); + nl_socket_free(sk); + + return rc == 1 ? 0 : rc; +} + static int dispatch_peer_ni_cmd(__u32 cmd, struct lnet_ioctl_peer_cfg *data, char *err_str, char *cmd_str) { diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index 8a482a7..47d5903 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -793,6 +793,20 @@ int lustre_lnet_parse_nid_range(struct nid_node *head, char *nidstr, void lustre_lnet_free_list(struct nid_node *head); /** + * yaml_lnet_configure + * + * Shared code for LNet setup and teardown. + * + * flags - Netlink flags that control setup behavior + * NLM_F_CREATE - initializes LNet stack + * NLM_F_EXCL - ignore module params + * 0 - tears down LNet stack + * + * msg error msg string returned. + */ +int yaml_lnet_configure(int flags, const char **msg); + +/** * yaml_emitter_set_output_netlink * * Special handling to integrate LNet handling into libyaml. diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index 299b367..4ee44b9 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -899,6 +899,11 @@ static void yaml_lnet_print_error(int op, char *cmd, const char *errstr) if (rc == 0) goto emitter_error; + if (strcmp(cmd, "lnet") == 0) { + flag = "configure"; + goto skip_op; + } + switch (op) { case NLM_F_CREATE: flag = "add"; @@ -917,7 +922,7 @@ static void yaml_lnet_print_error(int op, char *cmd, const char *errstr) flag = "show"; break; } - +skip_op: yaml_scalar_event_initialize(&event, NULL, (yaml_char_t *)YAML_STR_TAG, (yaml_char_t *)flag, @@ -1221,6 +1226,8 @@ static int jt_config_lnet(int argc, char **argv) { struct cYAML *err_rc = NULL; bool load_mod_params = false; + int flags = NLM_F_CREATE; + const char *msg = NULL; int rc, opt; const char *const short_options = "a"; @@ -1244,6 +1251,21 @@ static int jt_config_lnet(int argc, char **argv) } } + if (!load_mod_params) + flags |= NLM_F_EXCL; + + rc = yaml_lnet_configure(flags, &msg); + if (rc != -EOPNOTSUPP) { + if (rc < 0) { + char errstr[256]; + + snprintf(errstr, sizeof(errstr), + "LNet configure error: %s", msg); + yaml_lnet_print_error(flags, "lnet", errstr); + } + return rc; + } + rc = lustre_lnet_config_ni_system(LNET_CONFIGURE, load_mod_params, -1, &err_rc); @@ -1258,12 +1280,25 @@ static int jt_config_lnet(int argc, char **argv) static int jt_unconfig_lnet(int argc, char **argv) { struct cYAML *err_rc = NULL; + const char *msg = NULL; int rc; rc = check_cmd(lnet_cmds, "lnet", "unconfigure", 0, argc, argv); if (rc) return rc; + rc = yaml_lnet_configure(0, &msg); + if (rc != -EOPNOTSUPP) { + if (rc < 0) { + char errstr[256]; + + snprintf(errstr, sizeof(errstr), + "LNet configure error: %s", msg); + yaml_lnet_print_error(0, "lnet", errstr); + } + return rc; + } + rc = lustre_lnet_config_ni_system(LNET_UNCONFIGURE, 0, -1, &err_rc); if (rc != LUSTRE_CFG_RC_NO_ERR) diff --git a/lustre/utils/portals.c b/lustre/utils/portals.c index e9b7129..0e928bc 100644 --- a/lustre/utils/portals.c +++ b/lustre/utils/portals.c @@ -304,6 +304,7 @@ int jt_ptl_network(int argc, char **argv) { struct libcfs_ioctl_data data; __u32 net = LNET_NET_ANY; + const char *msg = NULL; int rc; if (argc != 2) { @@ -312,9 +313,26 @@ int jt_ptl_network(int argc, char **argv) } if (!strcmp(argv[1], "unconfigure") || !strcmp(argv[1], "down")) { + rc = yaml_lnet_configure(0, &msg); + if (rc != -EOPNOTSUPP) { + switch (rc) { + case 0: + printf("LNET ready to unload\n"); + break; + case -ENODEV: + case -EBUSY: + printf("%s\n", msg); + break; + default: + printf("LNET unconfigure error %u: %s\n", + -rc, msg); + break; + } + return rc; + } + LIBCFS_IOC_INIT(data); rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_UNCONFIGURE, &data); - if (rc == 0) { printf("LNET ready to unload\n"); return 0; @@ -332,6 +350,20 @@ int jt_ptl_network(int argc, char **argv) errno, strerror(errno)); return -1; } else if (!strcmp(argv[1], "configure") || !strcmp(argv[1], "up")) { + rc = yaml_lnet_configure(NLM_F_CREATE, &msg); + if (rc != -EOPNOTSUPP) { + switch (rc) { + case 0: + printf("LNET configured\n"); + break; + default: + fprintf(stderr, "LNET configure error %u: %s\n", + -rc, msg); + break; + } + return rc; + } + LIBCFS_IOC_INIT(data); rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_CONFIGURE, &data); -- 1.8.3.1