From 892f675e660a5afa0d93d883aab46bb54f107818 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Sun, 23 Jun 2019 19:27:42 -0500 Subject: [PATCH] LU-12410 lnet: Convert lnetctl peer add and del Convert the lnetctl peer add and del commands to utilize the new capabilities provided by the nidstrings library. Test-Parameters: trivial Test-Parameters: testlist=sanity-lnet Signed-off-by: Chris Horn Change-Id: I50693a2af6fef2e1ef3b34fd02c7423625cb7665 Reviewed-on: https://review.whamcloud.com/35305 Reviewed-by: Shaun Tancheff Tested-by: jenkins Tested-by: Maloo Reviewed-by: Petros Koutoupis Reviewed-by: Oleg Drokin --- lnet/utils/lnetconfig/liblnetconfig.c | 170 ++++++++++++++++++++++++--- lnet/utils/lnetconfig/liblnetconfig.h | 46 ++++++++ lnet/utils/lnetctl.c | 211 ++++++++++++---------------------- 3 files changed, 277 insertions(+), 150 deletions(-) diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 9f77bc9..5a7e172 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -560,23 +560,17 @@ int lustre_lnet_config_ni_system(bool up, bool load_ni_from_mod, return rc; } -static int dispatch_peer_ni_cmd(lnet_nid_t pnid, lnet_nid_t nid, __u32 cmd, - struct lnet_ioctl_peer_cfg *data, +static int dispatch_peer_ni_cmd(__u32 cmd, struct lnet_ioctl_peer_cfg *data, char *err_str, char *cmd_str) { int rc; - data->prcfg_prim_nid = pnid; - data->prcfg_cfg_nid = nid; - rc = l_ioctl(LNET_DEV_ID, cmd, data); - if (rc != 0) { + if (rc) { rc = -errno; - snprintf(err_str, - LNET_MAX_STR_LEN, - "\"cannot %s peer ni: %s\"", - (cmd_str) ? cmd_str : "add", strerror(errno)); - err_str[LNET_MAX_STR_LEN - 1] = '\0'; + snprintf(err_str, LNET_MAX_STR_LEN, + "\"%s peer ni operation failed: %s\"", + cmd_str, strerror(errno)); } return rc; @@ -884,6 +878,59 @@ out: return rc; } +static int lustre_lnet_handle_peer_nidlist(lnet_nid_t *nidlist, int num_nids, + bool is_mr, __u32 cmd, + char *cmd_type, char *err_str) +{ + struct lnet_ioctl_peer_cfg data; + int rc, nid_idx; + + if (cmd == IOC_LIBCFS_ADD_PEER_NI) { + /* When adding a peer we first need to create the peer using the + * specified (or implied) primary nid. Then we can add + * additional nids to this peer using the primary nid as a key + */ + LIBCFS_IOC_INIT_V2(data, prcfg_hdr); + data.prcfg_mr = is_mr; + data.prcfg_prim_nid = nidlist[0]; + data.prcfg_cfg_nid = LNET_NID_ANY; + + rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); + + if (rc) + return rc; + } + + /* Add or delete any specified NIs associated with the specified + * (or implied) primary nid + */ + for (nid_idx = 1; nid_idx < num_nids; nid_idx++) { + LIBCFS_IOC_INIT_V2(data, prcfg_hdr); + data.prcfg_mr = is_mr; + data.prcfg_prim_nid = nidlist[0]; + data.prcfg_cfg_nid = nidlist[nid_idx]; + + rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); + + if (rc) + return rc; + } + + if (cmd == IOC_LIBCFS_DEL_PEER_NI && num_nids == 1) { + /* In the delete case we may have been given just the + * primary nid of the peer. This tells us to delete the peer + * completely (rather than just delete some of its NIs) + */ + LIBCFS_IOC_INIT_V2(data, prcfg_hdr); + data.prcfg_prim_nid = nidlist[0]; + data.prcfg_cfg_nid = LNET_NID_ANY; + + rc = dispatch_peer_ni_cmd(cmd, &data, err_str, cmd_type); + } + + return rc; +} + static int lustre_lnet_handle_peer_ip2nets(char **nid, int num_nids, bool mr, bool range, __u32 cmd, char *cmd_type, char *err_str) @@ -944,14 +991,19 @@ static int lustre_lnet_handle_peer_ip2nets(char **nid, int num_nids, bool mr, else if (!range && i == 0) peer_nid = LNET_NID_ANY; + LIBCFS_IOC_INIT_V2(data, prcfg_hdr); + data.prcfg_mr = mr; + data.prcfg_prim_nid = prim_nid; + data.prcfg_cfg_nid = peer_nid; + /* * If prim_nid is not provided then the first nid in the * list becomes the prim_nid. First time round the loop * use LNET_NID_ANY for the first parameter, then use * nid[0] as the key nid after wards */ - rc = dispatch_peer_ni_cmd(prim_nid, peer_nid, cmd, - &data, err_str, cmd_type); + rc = dispatch_peer_ni_cmd(cmd, &data, err_str, + cmd_type); if (rc != 0) goto out; @@ -974,6 +1026,50 @@ out: return rc; } +int lustre_lnet_config_peer_nidlist(char *pnidstr, lnet_nid_t *lnet_nidlist, + int num_nids, bool is_mr, int seq_no, + struct cYAML **err_rc) +{ + int rc = LUSTRE_CFG_RC_NO_ERR; + char err_str[LNET_MAX_STR_LEN]; + lnet_nid_t *lnet_nidlist2 = NULL; + lnet_nid_t pnid; + + if (pnidstr) { + pnid = libcfs_str2nid(pnidstr); + if (pnid == LNET_NID_ANY) { + snprintf(err_str, LNET_MAX_STR_LEN, + "bad primary NID: '%s'", pnidstr); + rc = LUSTRE_CFG_RC_MISSING_PARAM; + goto out; + } + + num_nids++; + + lnet_nidlist2 = calloc(sizeof(*lnet_nidlist2), num_nids); + if (!lnet_nidlist2) { + snprintf(err_str, LNET_MAX_STR_LEN, "out of memory"); + rc = LUSTRE_CFG_RC_OUT_OF_MEM; + goto out; + } + lnet_nidlist2[0] = pnid; + memcpy(&lnet_nidlist2[1], lnet_nidlist, sizeof(*lnet_nidlist) * + (num_nids - 1)); + } + + rc = lustre_lnet_handle_peer_nidlist((pnidstr) ? lnet_nidlist2 : + lnet_nidlist, + num_nids, is_mr, + IOC_LIBCFS_ADD_PEER_NI, ADD_CMD, + err_str); +out: + if (lnet_nidlist2) + free(lnet_nidlist2); + + cYAML_build_error(rc, seq_no, ADD_CMD, "peer_ni", err_str, err_rc); + return rc; +} + int lustre_lnet_config_peer_nid(char *pnid, char **nid, int num_nids, bool mr, bool ip2nets, int seq_no, struct cYAML **err_rc) @@ -1028,6 +1124,54 @@ out: return rc; } +int lustre_lnet_del_peer_nidlist(char *pnidstr, lnet_nid_t *lnet_nidlist, + int num_nids, int seq_no, + struct cYAML **err_rc) +{ + int rc = LUSTRE_CFG_RC_NO_ERR; + char err_str[LNET_MAX_STR_LEN]; + lnet_nid_t *lnet_nidlist2 = NULL; + lnet_nid_t pnid; + + if (!pnidstr) { + snprintf(err_str, sizeof(err_str), + "\"Primary nid is not provided\""); + rc = LUSTRE_CFG_RC_MISSING_PARAM; + goto out; + } + + pnid = libcfs_str2nid(pnidstr); + if (pnid == LNET_NID_ANY) { + rc = LUSTRE_CFG_RC_BAD_PARAM; + snprintf(err_str, sizeof(err_str), + "bad key NID: '%s'", + pnidstr); + goto out; + } + + num_nids++; + lnet_nidlist2 = calloc(sizeof(*lnet_nidlist2), num_nids); + if (!lnet_nidlist2) { + snprintf(err_str, sizeof(err_str), + "out of memory"); + rc = LUSTRE_CFG_RC_OUT_OF_MEM; + goto out; + } + lnet_nidlist2[0] = pnid; + memcpy(&lnet_nidlist2[1], lnet_nidlist, sizeof(*lnet_nidlist) * + (num_nids - 1)); + + rc = lustre_lnet_handle_peer_nidlist(lnet_nidlist2, num_nids, false, + IOC_LIBCFS_DEL_PEER_NI, DEL_CMD, + err_str); +out: + if (lnet_nidlist2) + free(lnet_nidlist2); + + cYAML_build_error(rc, seq_no, DEL_CMD, "peer_ni", err_str, err_rc); + return rc; +} + int lustre_lnet_del_peer_nid(char *pnid, char **nid, int num_nids, bool ip2nets, int seq_no, struct cYAML **err_rc) { diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index 5c5fb6a..163a146 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -54,6 +54,12 @@ enum lnetctl_cmd { LNETCTL_LAST_CMD }; +/* + * Max number of nids we'll configure for a single peer via a single DLC + * operation + */ +#define LNET_MAX_NIDS_PER_PEER 128 + struct lnet_dlc_network_descr { struct list_head network_on_rule; __u32 nw_id; @@ -532,6 +538,30 @@ int lustre_lnet_config_peer_nid(char *pnid, char **nid, int num_nids, struct cYAML **err_rc); /* + * lustre_lnet_config_peer_nidlist + * Add a peer NID to a peer with primary NID pnid. If a pnid is not provided + * then the first NID in the NID list becomes the primary NID for a newly + * created peer. + * Otherwise, if the provided primary NID is unique, then a new peer is + * created with this primary NID, and the NIDs in the NID list are added as + * secondary NIDs to this new peer. + * If any of the NIDs in the NID list are not unique then the operation + * fails. Some peer NIDs might have already been added. It's the responsibility + * of the caller of this API to remove the added NIDs if so desired. + * + * pnid - The desired primary NID of a new peer, or the primary NID of + * an existing peer. + * lnet_nidlist - List of LNet NIDs to add to the peer + * num_nids - The number of LNet NIDs in the lnet_nidlist array + * mr - Specifies whether this peer is MR capable. + * seq_no - sequence number of the command + * err_rc - YAML structure of the resultant return code + */ +int lustre_lnet_config_peer_nidlist(char *pnid, lnet_nid_t *lnet_nidlist, + int num_nids, bool mr, int seq_no, + struct cYAML **err_rc); + +/* * lustre_lnet_del_peer_nid * Delete the nids given in the nid list from the peer with primary NID * pnid. If pnid is NULL or it doesn't identify a peer the operation @@ -549,6 +579,22 @@ int lustre_lnet_del_peer_nid(char *pnid, char **nid, int num_nids, bool ip2nets, int seq_no, struct cYAML **err_rc); /* + * lustre_lnet_del_peer_nidlist + * Delete the NIDs given in the NID list from the peer with the primary NID + * pnid. If pnid is NULL, or it doesn't identify a peer, the operation fails, + * and no change happens to the system. + * The operation is aborted on the first NID that fails to be deleted. + * + * pnid - The primary NID of the peer to be modified + * lnet_nidlist - The list of LNet NIDs to delete from the peer + * num_nids - the number of nids in the lnet_nidlist array + * seq_no - sequence number of the command + * err_rc - YAML structure of the resultant return code + */ +int lustre_lnet_del_peer_nidlist(char *pnid, lnet_nid_t *lnet_nidlist, + int num_nids, int seq_no, + struct cYAML **err_rc); +/* * lustre_lnet_show_peer * Show the peer identified by nid, knid. If knid is NULL all * peers in the system are shown. diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index 6bdce42..e4c9f19 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -1654,22 +1654,21 @@ static int jt_export(int argc, char **argv) return 0; } -static int jt_add_peer_nid(int argc, char **argv) +static int jt_peer_nid_common(int argc, char **argv, int cmd) { - char *prim_nid = NULL; - char **nids = NULL, **nids2 = NULL; - int size = 0; + int rc = LUSTRE_CFG_RC_NO_ERR, opt, num_nids, num_nidstrs, i; + bool is_mr = true; + char *prim_nid = NULL, *nidstr = NULL; + char err_str[LNET_MAX_STR_LEN] = "Error"; + char *nidstrarray[LNET_MAX_STR_LEN]; struct cYAML *err_rc = NULL; - int rc = LUSTRE_CFG_RC_NO_ERR, opt, i; - bool non_mr = false; - bool ip2nets = false, nid_list = false, prim_nid_present = false; + lnet_nid_t lnet_nidlist[LNET_MAX_NIDS_PER_PEER]; const char *const short_opts = "k:mn:"; const struct option long_opts[] = { { .name = "prim_nid", .has_arg = required_argument, .val = 'k' }, { .name = "non_mr", .has_arg = no_argument, .val = 'm' }, { .name = "nid", .has_arg = required_argument, .val = 'n' }, - { .name = "ip2nets", .has_arg = required_argument, .val = 'i' }, { .name = NULL } }; rc = check_cmd(peer_cmds, "peer", "add", 2, argc, argv); @@ -1680,161 +1679,99 @@ static int jt_add_peer_nid(int argc, char **argv) long_opts, NULL)) != -1) { switch (opt) { case 'k': - prim_nid_present = true; - if (ip2nets) { - cYAML_build_error(-1, -1, "peer", "add", - "ip2nets can not be specified" - " along side prim_nid parameter.", - &err_rc); - goto failed; - } prim_nid = optarg; break; - case 'i': case 'n': - if (opt == 'i') - ip2nets = true; - - if (opt == 'n') - nid_list = true; - - if (ip2nets && (nid_list || prim_nid_present)) { - cYAML_build_error(-1, -1, "peer", "add", - "ip2nets can not be specified" - " along side nid or prim_nid" - " parameters", &err_rc); - goto failed; - } - - size = lustre_lnet_parse_nids(optarg, nids, size, - &nids2); - if (nids2 == NULL) - goto failed; - nids = nids2; - rc = LUSTRE_CFG_RC_OUT_OF_MEM; + nidstr = optarg; break; case 'm': - non_mr = true; + if (cmd == LNETCTL_DEL_CMD) { + rc = LUSTRE_CFG_RC_BAD_PARAM; + snprintf(err_str, LNET_MAX_STR_LEN, + "Unrecognized option '-%c'", opt); + goto out; + } + is_mr = false; break; case '?': - print_help(peer_cmds, "peer", "add"); + print_help(peer_cmds, "peer", + cmd == LNETCTL_ADD_CMD ? "add" : "del"); default: return 0; } } - for (; optind < argc; optind++) { - size = lustre_lnet_parse_nids(argv[optind], nids, size, - &nids2); - if (nids2 == NULL) - goto failed; - nids = nids2; + if (!(nidstr || prim_nid)) { + rc = LUSTRE_CFG_RC_BAD_PARAM; + snprintf(err_str, LNET_MAX_STR_LEN, + "--prim_nid or --nid (or both) must be specified"); + goto out; } - rc = lustre_lnet_config_peer_nid(prim_nid, nids, size, - !non_mr, ip2nets, -1, &err_rc); + if (!nidstr) { + /* We were only provided a primary nid */ + num_nids = 0; + if (cmd == LNETCTL_ADD_CMD) + rc = lustre_lnet_config_peer_nidlist(prim_nid, + lnet_nidlist, + num_nids, is_mr, + -1, &err_rc); + else + rc = lustre_lnet_del_peer_nidlist(prim_nid, + lnet_nidlist, + num_nids, -1, + &err_rc); -failed: - if (nids) { - /* free the array of nids */ - for (i = 0; i < size; i++) - free(nids[i]); - free(nids); + goto out; } - if (rc != LUSTRE_CFG_RC_NO_ERR) + rc = tokenize_nidstr(nidstr, &nidstrarray[0], err_str); + if (rc <= 0) + goto out; + + num_nidstrs = rc; + + for (i = 0; i < num_nidstrs; i++) { + rc = lustre_lnet_parse_nidstr(nidstrarray[i], lnet_nidlist, + LNET_MAX_NIDS_PER_PEER, err_str); + if (rc < 0) + goto out; + + num_nids = rc; + + if (cmd == LNETCTL_ADD_CMD) + rc = lustre_lnet_config_peer_nidlist(prim_nid, + lnet_nidlist, + num_nids, is_mr, + -1, &err_rc); + else + rc = lustre_lnet_del_peer_nidlist(prim_nid, + lnet_nidlist, + num_nids, -1, + &err_rc); + } + +out: + if (rc != LUSTRE_CFG_RC_NO_ERR) { + cYAML_build_error(rc, -1, "peer", + cmd == LNETCTL_ADD_CMD ? "add" : "del", + err_str, &err_rc); cYAML_print_tree2file(stderr, err_rc); + } cYAML_free_tree(err_rc); return rc; } -static int jt_del_peer_nid(int argc, char **argv) +static int jt_add_peer_nid(int argc, char **argv) { - char *prim_nid = NULL; - char **nids = NULL, **nids2 = NULL; - struct cYAML *err_rc = NULL; - int rc = LUSTRE_CFG_RC_NO_ERR, opt, i, size = 0; - bool ip2nets = false, nid_list = false, prim_nid_present = false; - - const char *const short_opts = "k:n:"; - const struct option long_opts[] = { - { .name = "prim_nid", .has_arg = required_argument, .val = 'k' }, - { .name = "nid", .has_arg = required_argument, .val = 'n' }, - { .name = "ip2nets", .has_arg = required_argument, .val = 'i' }, - { .name = NULL } }; - - rc = check_cmd(peer_cmds, "peer", "del", 2, argc, argv); - if (rc) - return rc; - - while ((opt = getopt_long(argc, argv, short_opts, - long_opts, NULL)) != -1) { - switch (opt) { - case 'k': - prim_nid_present = true; - if (ip2nets) { - cYAML_build_error(-1, -1, "peer", "add", - "ip2nets can not be specified" - " along side prim_nid parameter.", - &err_rc); - goto failed; - } - prim_nid = optarg; - break; - case 'i': - case 'n': - if (opt == 'i') - ip2nets = true; - - if (opt == 'n') - nid_list = true; - - if (ip2nets && (nid_list || prim_nid_present)) { - cYAML_build_error(-1, -1, "peer", "add", - "ip2nets can not be specified" - " along side nid or prim_nid" - " parameters", &err_rc); - goto failed; - } - size = lustre_lnet_parse_nids(optarg, nids, size, - &nids2); - if (nids2 == NULL) - goto failed; - nids = nids2; - rc = LUSTRE_CFG_RC_OUT_OF_MEM; - break; - case '?': - print_help(peer_cmds, "peer", "del"); - default: - return 0; - } - } - - for (; optind < argc; optind++) { - size = lustre_lnet_parse_nids(argv[optind], nids, size, - &nids2); - if (nids2 == NULL) - goto failed; - nids = nids2; - } - - rc = lustre_lnet_del_peer_nid(prim_nid, nids, size, ip2nets, -1, &err_rc); - -failed: - if (nids) { - for (i = 0; i < size; i++) - free(nids[i]); - free(nids); - } - - if (rc != LUSTRE_CFG_RC_NO_ERR) - cYAML_print_tree2file(stderr, err_rc); - - cYAML_free_tree(err_rc); + return jt_peer_nid_common(argc, argv, LNETCTL_ADD_CMD); +} - return rc; +static int jt_del_peer_nid(int argc, char **argv) +{ + return jt_peer_nid_common(argc, argv, LNETCTL_DEL_CMD); } static int jt_show_peer(int argc, char **argv) -- 1.8.3.1