From adb73693529da52741af7f20b4ab8229e47e6713 Mon Sep 17 00:00:00 2001 From: Sonia Sharma Date: Wed, 20 Mar 2019 01:20:12 -0400 Subject: [PATCH] LU-9121 lnet: Add a selection policy This patch adds the function lustre_lnet_add_udsp() which marshals the input rules and makes an ioctl call to add those rules in the lnet structures. Change-Id: I5fa1e34d401b9eca58381e23cac2eb49c4dd3575 Test-Parameters: trivial testlist=lnet-selftest,sanity-lnet Signed-off-by: Amir Shehata Signed-off-by: Sonia Sharma Signed-off-by: Serguei Smirnov Reviewed-on: https://review.whamcloud.com/34529 Reviewed-by: Sebastien Buisson Tested-by: jenkins Tested-by: Maloo --- libcfs/libcfs/util/nidstrings.c | 134 ++++++++++++++++++++++++- lnet/include/uapi/linux/lnet/nidstr.h | 4 + lnet/lnet/nidstrings.c | 2 +- lnet/utils/lnetconfig/liblnetconfig.c | 40 +++++--- lnet/utils/lnetconfig/liblnetconfig.h | 40 ++++++++ lnet/utils/lnetconfig/liblnetconfig_udsp.c | 151 ++++++++++++++++++++++++++++- lnet/utils/lnetctl.c | 89 +++++++++++++++++ 7 files changed, 443 insertions(+), 17 deletions(-) diff --git a/libcfs/libcfs/util/nidstrings.c b/libcfs/libcfs/util/nidstrings.c index 3f3115f..dc996a4 100644 --- a/libcfs/libcfs/util/nidstrings.c +++ b/libcfs/libcfs/util/nidstrings.c @@ -374,7 +374,7 @@ libcfs_num_str2addr(const char *str, int nob, __u32 *addr) * \retval 0 if \a str parsed to numeric address * \retval errno otherwise */ -static int +int libcfs_num_parse(char *str, int len, struct list_head *list) { struct cfs_expr_list *el; @@ -917,6 +917,138 @@ parse_nidrange(struct cfs_lstr *src, struct list_head *nidlist) return 0; } +static __u32 +libcfs_net_str_len(const char *str) +{ + int i; + struct netstrfns *nf = NULL; + + for (i = 0; i < libcfs_nnetstrfns; i++) { + nf = &libcfs_netstrfns[i]; + if (!strncmp(str, nf->nf_name, strlen(nf->nf_name))) + return strlen(nf->nf_name); + } + + return 0; +} + +int +parse_net_range(char *str, __u32 len, struct list_head *net_num, + __u32 *net_type) +{ + struct cfs_lstr next; + __u32 net_type_len; + __u32 net; + char *bracket; + char *star; + + if (!str) + return -EINVAL; + + next.ls_str = str; + next.ls_len = len; + + net_type_len = libcfs_net_str_len(str); + + if (net_type_len < len) { + char c = str[net_type_len]; + + str[net_type_len] = '\0'; + net = libcfs_str2net(str); + str[net_type_len] = c; + } else { + net = libcfs_str2net(str); + } + + if (net == LNET_NIDNET(LNET_NID_ANY)) + return -EINVAL; + + *net_type = LNET_NETTYP(net); + + /* + * the net is either followed with an absolute number, *, or an + * expression enclosed in [] + */ + bracket = strchr(next.ls_str, '['); + star = strchr(next.ls_str, '*'); + + /* "*[" pattern not allowed */ + if (bracket && star && star < bracket) + return -EINVAL; + + if (!bracket) { + next.ls_str = str + net_type_len; + next.ls_len = strlen(next.ls_str); + } else { + next.ls_str = bracket; + next.ls_len = strlen(bracket); + } + + /* if there is no net number just return */ + if (next.ls_len == 0) + return 0; + + return libcfs_num_parse(next.ls_str, next.ls_len, + net_num); +} + +int +parse_address(struct cfs_lstr *src, const __u32 net_type, + struct list_head *addr) +{ + int i; + struct netstrfns *nf = NULL; + + for (i = 0; i < libcfs_nnetstrfns; i++) { + nf = &libcfs_netstrfns[i]; + if (net_type == nf->nf_type) + return nf->nf_parse_addrlist(src->ls_str, src->ls_len, + addr); + } + + return -EINVAL; +} + +int +cfs_parse_nid_parts(char *str, struct list_head *addr, + struct list_head *net_num, __u32 *net_type) +{ + struct cfs_lstr next; + struct cfs_lstr addrrange; + bool found = false; + int rc; + + if (!str) + return -EINVAL; + + next.ls_str = str; + next.ls_len = strlen(str); + + rc = cfs_gettok(&next, '@', &addrrange); + if (!rc) + return -EINVAL; + + if (!next.ls_str) { + /* only net is present */ + next.ls_str = str; + next.ls_len = strlen(str); + } else { + found = true; + } + + /* assume only net is present */ + rc = parse_net_range(next.ls_str, next.ls_len, net_num, net_type); + + /* + * if we successfully parsed the net range and there is no + * address, or if we fail to parse the net range then return + */ + if ((!rc && !found) || rc) + return rc; + + return parse_address(&addrrange, *net_type, addr); +} + /** * Frees addrrange structures of \a list. * diff --git a/lnet/include/uapi/linux/lnet/nidstr.h b/lnet/include/uapi/linux/lnet/nidstr.h index a02aa16..1e4da91 100644 --- a/lnet/include/uapi/linux/lnet/nidstr.h +++ b/lnet/include/uapi/linux/lnet/nidstr.h @@ -87,9 +87,12 @@ static inline char *libcfs_nid2str(lnet_nid_t nid) __u32 libcfs_str2net(const char *str); lnet_nid_t libcfs_str2nid(const char *str); int libcfs_str2anynid(lnet_nid_t *nid, const char *str); +int libcfs_num_parse(char *str, int len, struct list_head *list); char *libcfs_id2str(struct lnet_process_id id); void cfs_free_nidlist(struct list_head *list); int cfs_parse_nidlist(char *str, int len, struct list_head *list); +int cfs_parse_nid_parts(char *str, struct list_head *addr, + struct list_head *net_num, __u32 *net_type); int cfs_print_nidlist(char *buffer, int count, struct list_head *list); int cfs_match_nid(lnet_nid_t nid, struct list_head *list); int cfs_expand_nidlist(struct list_head *nidlist, lnet_nid_t *lnet_nidlist, @@ -103,6 +106,7 @@ int cfs_ip_addr_parse(char *str, int len, struct list_head *list); int cfs_ip_addr_match(__u32 addr, struct list_head *list); int cfs_nidrange_find_min_max(struct list_head *nidlist, char *min_nid, char *max_nid, __kernel_size_t nidstr_length); +void cfs_expr_list_free_list(struct list_head *list); struct netstrfns { __u32 nf_type; diff --git a/lnet/lnet/nidstrings.c b/lnet/lnet/nidstrings.c index e392133..df82695 100644 --- a/lnet/lnet/nidstrings.c +++ b/lnet/lnet/nidstrings.c @@ -607,7 +607,7 @@ libcfs_num_str2addr(const char *str, int nob, __u32 *addr) * \retval 0 if \a str parsed to numeric address * \retval errno otherwise */ -static int +int libcfs_num_parse(char *str, int len, struct list_head *list) { struct cfs_expr_list *el; diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index a5a3eeb..1c253c5 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -55,20 +55,6 @@ #include #include -#define CONFIG_CMD "configure" -#define UNCONFIG_CMD "unconfigure" -#define ADD_CMD "add" -#define DEL_CMD "del" -#define SHOW_CMD "show" -#define DBG_CMD "dbg" -#define MANAGE_CMD "manage" - -#define MAX_NUM_IPS 128 - -#define modparam_path "/sys/module/lnet/parameters/" -#define o2ib_modparam_path "/sys/module/ko2iblnd/parameters/" -#define gni_nid_path "/proc/cray_xt/" - #ifndef HAVE_USRSPC_RDMA_PS_TCP #define RDMA_PS_TCP 0x0106 #endif @@ -4548,6 +4534,31 @@ static int handle_yaml_show_numa(struct cYAML *tree, struct cYAML **show_rc, show_rc, err_rc); } +static int handle_yaml_config_udsp(struct cYAML *tree, struct cYAML **show_rc, + struct cYAML **err_rc) +{ + struct cYAML *seq_no, *src, *rte, *dst, *prio, *idx; + union lnet_udsp_action action; + + seq_no = cYAML_get_object_item(tree, "seq_no"); + src = cYAML_get_object_item(tree, "src"); + rte = cYAML_get_object_item(tree, "rte"); + dst = cYAML_get_object_item(tree, "dst"); + prio = cYAML_get_object_item(tree, "priority"); + idx = cYAML_get_object_item(tree, "idx"); + + action.udsp_priority = prio ? prio->cy_valueint : -1; + + return lustre_lnet_add_udsp(src ? src->cy_valuestring : NULL, + dst ? dst->cy_valuestring : NULL, + rte ? rte->cy_valuestring : NULL, + prio ? "priority" : "", + &action, + idx ? idx->cy_valueint : -1, + seq_no ? seq_no->cy_valueint : -1, + err_rc); +} + static int handle_yaml_config_global_settings(struct cYAML *tree, struct cYAML **show_rc, struct cYAML **err_rc) @@ -4810,6 +4821,7 @@ static struct lookup_cmd_hdlr_tbl lookup_config_tbl[] = { { .name = "numa", .cb = handle_yaml_config_numa }, { .name = "ping", .cb = handle_yaml_no_op }, { .name = "discover", .cb = handle_yaml_no_op }, + { .name = "udsp", .cb = handle_yaml_config_udsp }, { .name = NULL } }; static struct lookup_cmd_hdlr_tbl lookup_del_tbl[] = { diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index d9087fb..b3d5b0f 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -42,6 +42,21 @@ #define LUSTRE_CFG_RC_MATCH -7 #define LUSTRE_CFG_RC_SKIP -8 #define LUSTRE_CFG_RC_LAST_ELEM -9 +#define LUSTRE_CFG_RC_MARSHAL_FAIL -10 + +#define CONFIG_CMD "configure" +#define UNCONFIG_CMD "unconfigure" +#define ADD_CMD "add" +#define DEL_CMD "del" +#define SHOW_CMD "show" +#define DBG_CMD "dbg" +#define MANAGE_CMD "manage" + +#define MAX_NUM_IPS 128 + +#define modparam_path "/sys/module/lnet/parameters/" +#define o2ib_modparam_path "/sys/module/ko2iblnd/parameters/" +#define gni_nid_path "/proc/cray_xt/" enum lnetctl_cmd { LNETCTL_CONFIG_CMD = 1, @@ -114,6 +129,15 @@ struct lnet_udsp { } udsp_action; }; +/* This union is passed from lnetctl to fill the action union in udsp + * structure + * TODO: The idea here is if we add extra actions, ex: drop, it can be + * added to the union + */ +union lnet_udsp_action { + int udsp_priority; +}; + /* forward declaration of the cYAML structure. */ struct cYAML; @@ -746,4 +770,20 @@ int lustre_lnet_parse_interfaces(char *intf_str, int lustre_lnet_parse_nidstr(char *nidstr, lnet_nid_t *lnet_nidlist, int max_nids, char *err_str); +/* lustre_lnet_add_udsp + * Add a selection policy. + * src - source NID descriptor + * dst - destination NID descriptor + * rte - router NID descriptor + * type - action type + * action - union of the action + * idx - the index to delete + * seq_no - sequence number of the request + * err_rc - [OUT] struct cYAML tree describing the error. Freed by + * caller + */ +int lustre_lnet_add_udsp(char *src, char *dst, char *rte, char *type, + union lnet_udsp_action *action, int idx, + int seq_no, struct cYAML **err_rc); + #endif /* LIB_LNET_CONFIG_API_H */ diff --git a/lnet/utils/lnetconfig/liblnetconfig_udsp.c b/lnet/utils/lnetconfig/liblnetconfig_udsp.c index 8c0e6dc..873940e 100644 --- a/lnet/utils/lnetconfig/liblnetconfig_udsp.c +++ b/lnet/utils/lnetconfig/liblnetconfig_udsp.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "liblnetconfig.h" static inline bool @@ -408,7 +409,7 @@ copy_nid_range(struct lnet_ud_nid_descr *nid_descr, char *type, return 0; } -int +static int lnet_udsp_marshal(struct lnet_udsp *udsp, void *bulk, __s32 bulk_size) { @@ -443,3 +444,151 @@ lnet_udsp_marshal(struct lnet_udsp *udsp, void *bulk, return rc; } +static enum lnet_udsp_action_type +lnet_str2udsp_action(char *type) +{ + if (!type) + return EN_LNET_UDSP_ACTION_NONE; + + if (!strncmp(type, "priority", strlen("priority"))) + return EN_LNET_UDSP_ACTION_PRIORITY; + + if (!strncmp(type, "pref", strlen("pref"))) + return EN_LNET_UDSP_ACTION_PREFERRED_LIST; + + return EN_LNET_UDSP_ACTION_NONE; +} + +int lustre_lnet_add_udsp(char *src, char *dst, char *rte, + char *type, union lnet_udsp_action *action, + int idx, int seq_no, struct cYAML **err_rc) +{ + struct lnet_udsp *udsp = NULL; + struct lnet_ioctl_udsp *udsp_bulk; + int rc = LUSTRE_CFG_RC_OUT_OF_MEM; + void *bulk = NULL; + __u32 bulk_size; + char err_str[LNET_MAX_STR_LEN]; + enum lnet_udsp_action_type action_type; + + snprintf(err_str, sizeof(err_str), "\"success\""); + + action_type = lnet_str2udsp_action(type); + if (action_type == EN_LNET_UDSP_ACTION_NONE) { + snprintf(err_str, sizeof(err_str), + "\"bad action type specified: %s\"", type); + rc = LUSTRE_CFG_RC_BAD_PARAM; + goto out; + } + + /* sanitize parameters: + * src-dst can be simultaneously present + * dst-rte can be simultaneously present + */ + if ((!src && !rte && !dst) || + (src && rte && dst) || + (src && rte && !dst)) { + snprintf(err_str, sizeof(err_str), + "\"The combination of src, dst and rte is not supported\""); + rc = LUSTRE_CFG_RC_BAD_PARAM; + goto out; + } + + udsp = lnet_udsp_alloc(); + if (!udsp) { + snprintf(err_str, sizeof(err_str), "\"out of memory\""); + goto out; + } + + udsp->udsp_idx = idx; + udsp->udsp_action_type = action_type; + + /* a priority of -1 will result in the lowest possible priority */ + if (action_type == EN_LNET_UDSP_ACTION_PRIORITY) + udsp->udsp_action.udsp_priority = action->udsp_priority; + + /* override with the default + * if priority is expected, but not specified + */ + if (!rte && ((dst && !src) || (src && !dst)) && + action_type != EN_LNET_UDSP_ACTION_PRIORITY) { + udsp->udsp_action_type = EN_LNET_UDSP_ACTION_PRIORITY; + udsp->udsp_action.udsp_priority = 0; + } + + if (src) { + rc = cfs_parse_nid_parts(src, &udsp->udsp_src.ud_addr_range, + &udsp->udsp_src.ud_net_id.udn_net_num_range, + &udsp->udsp_src.ud_net_id.udn_net_type); + if (rc < 0) { + snprintf(err_str, + sizeof(err_str), + "\failed to parse src parameter\""); + goto out; + } + } + if (dst) { + rc = cfs_parse_nid_parts(dst, &udsp->udsp_dst.ud_addr_range, + &udsp->udsp_dst.ud_net_id.udn_net_num_range, + &udsp->udsp_dst.ud_net_id.udn_net_type); + if (rc < 0) { + snprintf(err_str, + sizeof(err_str), + "\failed to parse dst parameter\""); + goto out; + } + } + if (rte) { + rc = cfs_parse_nid_parts(rte, &udsp->udsp_rte.ud_addr_range, + &udsp->udsp_rte.ud_net_id.udn_net_num_range, + &udsp->udsp_rte.ud_net_id.udn_net_type); + if (rc < 0) { + snprintf(err_str, + sizeof(err_str), + "\failed to parse rte parameter\""); + goto out; + } + } + + bulk_size = lnet_get_udsp_size(udsp); + bulk = calloc(1, bulk_size); + if (!bulk) { + rc = LUSTRE_CFG_RC_OUT_OF_MEM; + snprintf(err_str, sizeof(err_str), "\"out of memory\""); + goto out; + } + + udsp_bulk = bulk; + LIBCFS_IOC_INIT_V2(*udsp_bulk, iou_hdr); + udsp_bulk->iou_hdr.ioc_len = bulk_size; + udsp_bulk->iou_bulk_size = bulk_size - sizeof(*udsp_bulk); + + rc = lnet_udsp_marshal(udsp, bulk, bulk_size); + if (rc != LUSTRE_CFG_RC_NO_ERR) { + rc = LUSTRE_CFG_RC_MARSHAL_FAIL; + snprintf(err_str, + sizeof(err_str), + "\"failed to marshal udsp\""); + goto out; + } + + udsp_bulk->iou_bulk = bulk + sizeof(*udsp_bulk); + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_UDSP, bulk); + if (rc < 0) { + rc = errno; + snprintf(err_str, sizeof(err_str), + "\"cannot add udsp: %s\"", strerror(errno)); + goto out; + } + + rc = LUSTRE_CFG_RC_NO_ERR; + +out: + if (bulk) + free(bulk); + if (udsp) + lnet_udsp_free(udsp, false); + cYAML_build_error(rc, seq_no, ADD_CMD, "udsp", err_str, err_rc); + return rc; +} diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index 2328d00..631c18e 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -65,6 +65,7 @@ static int jt_set_max_intf(int argc, char **argv); static int jt_set_discovery(int argc, char **argv); static int jt_set_drop_asym_route(int argc, char **argv); static int jt_list_peer(int argc, char **argv); +static int jt_add_udsp(int argc, char **argv); /*static int jt_show_peer(int argc, char **argv);*/ static int lnetctl_list_commands(int argc, char **argv); static int jt_import(int argc, char **argv); @@ -85,6 +86,7 @@ static int jt_set_peer_ni_value(int argc, char **argv); static int jt_calc_service_id(int argc, char **argv); static int jt_set_response_tracking(int argc, char **argv); static int jt_set_recovery_limit(int argc, char **argv); +static int jt_udsp(int argc, char **argv); command_t cmd_list[] = { {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all]"}, @@ -106,6 +108,7 @@ command_t cmd_list[] = { {"ping", jt_ping, 0, "ping nid,[nid,...]"}, {"discover", jt_discover, 0, "discover nid[,nid,...]"}, {"service-id", jt_calc_service_id, 0, "Calculate IB Lustre service ID\n"}, + {"udsp", jt_udsp, 0, "udsp {add | help}"}, {"help", Parser_help, 0, "help"}, {"exit", Parser_quit, 0, "quit"}, {"quit", Parser_quit, 0, "quit"}, @@ -259,6 +262,17 @@ command_t peer_cmds[] = { { 0, 0, 0, NULL } }; +command_t udsp_cmds[] = { + {"add", jt_add_udsp, 0, "add a udsp\n" + "\t--src: ip2nets syntax specifying the local NID to match\n" + "\t--dst: ip2nets syntax specifying the remote NID to match\n" + "\t--rte: ip2nets syntax specifying the router NID to match\n" + "\t--priority: priority value (0 - highest priority)\n" + "\t--idx: index of where to insert the rule.\n" + "\t By default, appends to the end of the rule list.\n"}, + { 0, 0, 0, NULL } +}; + static int jt_calc_service_id(int argc, char **argv) { int rc; @@ -1570,6 +1584,17 @@ static int jt_set(int argc, char **argv) return Parser_execarg(argc - 1, &argv[1], set_cmds); } +static int jt_udsp(int argc, char **argv) +{ + int rc; + + rc = check_cmd(udsp_cmds, "udsp", NULL, 2, argc, argv); + if (rc) + return rc; + + return Parser_execarg(argc - 1, &argv[1], udsp_cmds); +} + static int jt_import(int argc, char **argv) { char *file = NULL; @@ -2054,6 +2079,70 @@ static int jt_discover(int argc, char **argv) return rc; } +static int jt_add_udsp(int argc, char **argv) +{ + char *src = NULL, *dst = NULL, *rte = NULL; + struct cYAML *err_rc = NULL; + union lnet_udsp_action udsp_action; + long int idx = -1, priority = -1; + int opt, rc = 0; + char *action_type = "pref"; + + const char *const short_options = "s:d:r:p:i:"; + static const struct option long_options[] = { + { .name = "src", .has_arg = required_argument, .val = 's' }, + { .name = "dst", .has_arg = required_argument, .val = 'd' }, + { .name = "rte", .has_arg = required_argument, .val = 'r' }, + { .name = "priority", .has_arg = required_argument, .val = 'p' }, + { .name = "idx", .has_arg = required_argument, .val = 'i' }, + { .name = NULL } }; + + rc = check_cmd(udsp_cmds, "udsp", "add", 0, argc, argv); + if (rc) + return rc; + + while ((opt = getopt_long(argc, argv, short_options, + long_options, NULL)) != -1) { + switch (opt) { + case 's': + src = optarg; + break; + case 'd': + dst = optarg; + break; + case 'r': + rte = optarg; + break; + case 'p': + rc = parse_long(optarg, &priority); + if (rc != 0) + priority = -1; + action_type = "priority"; + udsp_action.udsp_priority = priority; + break; + case 'i': + rc = parse_long(optarg, &idx); + if (rc != 0) + idx = 0; + break; + case '?': + print_help(udsp_cmds, "udsp", "add"); + default: + return 0; + } + } + + rc = lustre_lnet_add_udsp(src, dst, rte, action_type, &udsp_action, + idx, -1, &err_rc); + + if (rc != LUSTRE_CFG_RC_NO_ERR) + cYAML_print_tree2file(stderr, err_rc); + + cYAML_free_tree(err_rc); + + return rc; +} + static int lnetctl_list_commands(int argc, char **argv) { char buffer[81] = ""; /* 80 printable chars + terminating NUL */ -- 1.8.3.1