From 00e73d22f04f440ac81281f584445e258269f654 Mon Sep 17 00:00:00 2001 From: Olaf Weber Date: Fri, 27 Jan 2017 16:15:24 +0100 Subject: [PATCH] LU-9480 lnet: configure lnet_interfaces_max tunable from dlc Added the ability to configure lnet_interfaces_max from DLC. Combined the configure and show of numa range and max interfaces under a "global" YAML element when configuring using YAML. Test-Parameters: trivial Signed-off-by: Amir Shehata Signed-off-by: Olaf Weber Change-Id: I6f8babdf7900f963cd86acf92468175a49bbaeee Reviewed-on: https://review.whamcloud.com/25771 --- lnet/include/uapi/linux/lnet/lnet-dlc.h | 6 +- lnet/lnet/api-ni.c | 16 +- lnet/utils/lnetconfig/liblnetconfig.c | 253 +++++++++++++++++++++++++++----- lnet/utils/lnetconfig/liblnetconfig.h | 24 +++ lnet/utils/lnetctl.c | 70 +++++++-- 5 files changed, 311 insertions(+), 58 deletions(-) diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h index ea54f9f..ff2eaf7 100644 --- a/lnet/include/uapi/linux/lnet/lnet-dlc.h +++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h @@ -239,9 +239,9 @@ struct lnet_ioctl_peer_cfg { void __user *prcfg_bulk; }; -struct lnet_ioctl_numa_range { - struct libcfs_ioctl_hdr nr_hdr; - __u32 nr_range; +struct lnet_ioctl_set_value { + struct libcfs_ioctl_hdr sv_hdr; + __u32 sv_value; }; struct lnet_ioctl_lnet_stats { diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 2db2d7a..8e20d36 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -2784,22 +2784,22 @@ LNetCtl(unsigned int cmd, void *arg) return rc; case IOC_LIBCFS_SET_NUMA_RANGE: { - struct lnet_ioctl_numa_range *numa; + struct lnet_ioctl_set_value *numa; numa = arg; - if (numa->nr_hdr.ioc_len != sizeof(*numa)) + if (numa->sv_hdr.ioc_len != sizeof(*numa)) return -EINVAL; - mutex_lock(&the_lnet.ln_api_mutex); - lnet_numa_range = numa->nr_range; - mutex_unlock(&the_lnet.ln_api_mutex); + lnet_net_lock(LNET_LOCK_EX); + lnet_numa_range = numa->sv_value; + lnet_net_unlock(LNET_LOCK_EX); return 0; } case IOC_LIBCFS_GET_NUMA_RANGE: { - struct lnet_ioctl_numa_range *numa; + struct lnet_ioctl_set_value *numa; numa = arg; - if (numa->nr_hdr.ioc_len != sizeof(*numa)) + if (numa->sv_hdr.ioc_len != sizeof(*numa)) return -EINVAL; - numa->nr_range = lnet_numa_range; + numa->sv_value = lnet_numa_range; return 0; } diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 885ef06..c182e6c 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -47,6 +47,8 @@ #include #include "liblnd.h" #include +#include +#include #include #include "liblnetconfig.h" #include "cyaml.h" @@ -58,6 +60,8 @@ #define SHOW_CMD "show" #define DBG_CMD "dbg" +#define modparam_path "/sys/module/lnet/parameters/" + /* * lustre_lnet_ip_range_descr * Describes an IP range. @@ -77,6 +81,62 @@ struct lustre_lnet_ip2nets { struct list_head ip2nets_ip_ranges; }; +int open_sysfs_file(const char *path, const char *attr, const int mode) +{ + int fd; + char filename[LNET_MAX_STR_LEN]; + + if (strlen(path) + strlen(attr) >= LNET_MAX_STR_LEN) + return -1; + + snprintf(filename, sizeof(filename), "%s%s", + path, attr); + + fd = open(filename, mode); + + return fd; +} + +static int read_sysfs_file(const char *path, const char *attr, + void *val, const size_t size, const int nelem) +{ + int fd; + int rc = LUSTRE_CFG_RC_GENERIC_ERR; + + fd = open_sysfs_file(path, attr, O_RDONLY); + if (fd == -1) + return LUSTRE_CFG_RC_NO_MATCH; + + if (read(fd, val, size * nelem) == -1) + goto close_fd; + + rc = LUSTRE_CFG_RC_NO_ERR; + +close_fd: + close(fd); + return rc; +} + +static int write_sysfs_file(const char *path, const char *attr, + void *val, const size_t size, const int nelem) +{ + int fd; + int rc = LUSTRE_CFG_RC_GENERIC_ERR; + + fd = open_sysfs_file(path, attr, O_WRONLY | O_TRUNC); + if (fd == -1) + return LUSTRE_CFG_RC_NO_MATCH; + + if (write(fd, val, size * nelem) == -1) + goto close_fd; + + rc = LUSTRE_CFG_RC_NO_ERR; + +close_fd: + close(fd); + return rc; +} + /* * free_intf_descr * frees the memory allocated for an intf descriptor. @@ -1777,9 +1837,31 @@ out: return rc; } +int lustre_lnet_config_max_intf(int max, int seq_no, struct cYAML **err_rc) +{ + int rc = LUSTRE_CFG_RC_NO_ERR; + char err_str[LNET_MAX_STR_LEN]; + char val[LNET_MAX_STR_LEN]; + + snprintf(err_str, sizeof(err_str), "\"success\""); + + snprintf(val, sizeof(val), "%d", max); + + rc = write_sysfs_file(modparam_path, "lnet_interfaces_max", val, + 1, strlen(val) + 1); + if (rc) + snprintf(err_str, sizeof(err_str), + "\"cannot configure max interfaces: %s\"", + strerror(errno)); + + cYAML_build_error(rc, seq_no, ADD_CMD, "max_interfaces", err_str, err_rc); + + return rc; +} + int lustre_lnet_config_numa_range(int range, int seq_no, struct cYAML **err_rc) { - struct lnet_ioctl_numa_range data; + struct lnet_ioctl_set_value data; int rc = LUSTRE_CFG_RC_NO_ERR; char err_str[LNET_MAX_STR_LEN]; @@ -1793,15 +1875,15 @@ int lustre_lnet_config_numa_range(int range, int seq_no, struct cYAML **err_rc) goto out; } - LIBCFS_IOC_INIT_V2(data, nr_hdr); - data.nr_range = range; + LIBCFS_IOC_INIT_V2(data, sv_hdr); + data.sv_value = range; rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_SET_NUMA_RANGE, &data); if (rc != 0) { rc = -errno; snprintf(err_str, sizeof(err_str), - "\"cannot configure buffers: %s\"", strerror(errno)); + "\"cannot configure numa_range: %s\"", strerror(errno)); goto out; } @@ -2211,18 +2293,83 @@ out: return rc; } +static void add_to_global(struct cYAML *show_rc, struct cYAML *node, + struct cYAML *root) +{ + struct cYAML *show_node; + + show_node = cYAML_get_object_item(show_rc, "global"); + if (show_node != NULL) + cYAML_insert_sibling(show_node->cy_child, + node->cy_child); + else + cYAML_insert_sibling(show_rc->cy_child, + node); + free(root); +} + +int lustre_lnet_show_max_intf(int seq_no, struct cYAML **show_rc, + struct cYAML **err_rc) +{ + int rc = LUSTRE_CFG_RC_OUT_OF_MEM; + char val[LNET_MAX_STR_LEN]; + int max_intf; + char err_str[LNET_MAX_STR_LEN]; + struct cYAML *root = NULL, *global = NULL; + + snprintf(err_str, sizeof(err_str), "\"out of memory\""); + + rc = read_sysfs_file(modparam_path, "lnet_interfaces_max", val, + 1, LNET_MAX_STR_LEN); + if (rc) { + snprintf(err_str, sizeof(err_str), + "\"cannot get max interfaces: %d\"", rc); + goto out; + } + + max_intf = atoi(val); + + root = cYAML_create_object(NULL, NULL); + if (root == NULL) + goto out; + + global = cYAML_create_object(root, "global"); + if (global == NULL) + goto out; + + if (cYAML_create_number(global, "max_intf", + max_intf) == NULL) + goto out; + + if (show_rc == NULL) + cYAML_print_tree(root); + + snprintf(err_str, sizeof(err_str), "\"success\""); +out: + if (show_rc == NULL || rc != LUSTRE_CFG_RC_NO_ERR) + cYAML_free_tree(root); + else if (show_rc != NULL && *show_rc != NULL) + add_to_global(*show_rc, global, root); + else + *show_rc = root; + + cYAML_build_error(rc, seq_no, SHOW_CMD, "global", err_str, err_rc); + + return rc; +} + int lustre_lnet_show_numa_range(int seq_no, struct cYAML **show_rc, struct cYAML **err_rc) { - struct lnet_ioctl_numa_range data; + struct lnet_ioctl_set_value data; int rc; int l_errno; char err_str[LNET_MAX_STR_LEN]; - struct cYAML *root = NULL, *range = NULL; + struct cYAML *root = NULL, *global = NULL; snprintf(err_str, sizeof(err_str), "\"out of memory\""); - LIBCFS_IOC_INIT_V2(data, nr_hdr); + LIBCFS_IOC_INIT_V2(data, sv_hdr); rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_GET_NUMA_RANGE, &data); if (rc != 0) { @@ -2241,12 +2388,12 @@ int lustre_lnet_show_numa_range(int seq_no, struct cYAML **show_rc, if (root == NULL) goto out; - range = cYAML_create_object(root, "numa"); - if (range == NULL) + global = cYAML_create_object(root, "global"); + if (global == NULL) goto out; - if (cYAML_create_number(range, "range", - data.nr_range) == NULL) + if (cYAML_create_number(global, "numa_range", + data.sv_value) == NULL) goto out; if (show_rc == NULL) @@ -2258,14 +2405,12 @@ out: if (show_rc == NULL || rc != LUSTRE_CFG_RC_NO_ERR) { cYAML_free_tree(root); } else if (show_rc != NULL && *show_rc != NULL) { - cYAML_insert_sibling((*show_rc)->cy_child, - root->cy_child); - free(root); + add_to_global(*show_rc, global, root); } else { *show_rc = root; } - cYAML_build_error(rc, seq_no, SHOW_CMD, "numa", err_str, err_rc); + cYAML_build_error(rc, seq_no, SHOW_CMD, "global", err_str, err_rc); return rc; } @@ -3029,39 +3174,77 @@ static int handle_yaml_show_stats(struct cYAML *tree, struct cYAML **show_rc, show_rc, err_rc); } -static int handle_yaml_config_numa(struct cYAML *tree, struct cYAML **show_rc, - struct cYAML **err_rc) +static int handle_yaml_config_global_settings(struct cYAML *tree, + struct cYAML **show_rc, + struct cYAML **err_rc) { - struct cYAML *seq_no, *range; + struct cYAML *max_intf, *numa, *seq_no; + int rc = 0; seq_no = cYAML_get_object_item(tree, "seq_no"); - range = cYAML_get_object_item(tree, "range"); + max_intf = cYAML_get_object_item(tree, "max_intf"); + if (max_intf) + rc = lustre_lnet_config_max_intf(max_intf->cy_valueint, + seq_no ? seq_no->cy_valueint + : -1, + err_rc); + + numa = cYAML_get_object_item(tree, "numa_range"); + if (numa) + rc = lustre_lnet_config_numa_range(numa->cy_valueint, + seq_no ? seq_no->cy_valueint + : -1, + err_rc); - return lustre_lnet_config_numa_range(range ? range->cy_valueint : -1, - seq_no ? seq_no->cy_valueint : -1, - err_rc); + return rc; } -static int handle_yaml_del_numa(struct cYAML *tree, struct cYAML **show_rc, - struct cYAML **err_rc) +static int handle_yaml_del_global_settings(struct cYAML *tree, + struct cYAML **show_rc, + struct cYAML **err_rc) { - struct cYAML *seq_no; + struct cYAML *max_intf, *numa, *seq_no; + int rc = 0; seq_no = cYAML_get_object_item(tree, "seq_no"); + max_intf = cYAML_get_object_item(tree, "max_intf"); + if (max_intf) + rc = lustre_lnet_config_max_intf(LNET_INTERFACES_MAX_DEFAULT, + seq_no ? seq_no->cy_valueint + : -1, + err_rc); + + numa = cYAML_get_object_item(tree, "numa_range"); + if (numa) + rc = lustre_lnet_config_numa_range(0, + seq_no ? seq_no->cy_valueint + : -1, + err_rc); - return lustre_lnet_config_numa_range(0, seq_no ? seq_no->cy_valueint : -1, - err_rc); + return rc; } -static int handle_yaml_show_numa(struct cYAML *tree, struct cYAML **show_rc, - struct cYAML **err_rc) +static int handle_yaml_show_global_settings(struct cYAML *tree, + struct cYAML **show_rc, + struct cYAML **err_rc) { - struct cYAML *seq_no; + struct cYAML *max_intf, *numa, *seq_no; + int rc = 0; seq_no = cYAML_get_object_item(tree, "seq_no"); + max_intf = cYAML_get_object_item(tree, "max_intf"); + if (max_intf) + rc = lustre_lnet_show_max_intf(seq_no ? seq_no->cy_valueint + : -1, + show_rc, err_rc); + + numa = cYAML_get_object_item(tree, "numa_range"); + if (numa) + rc = lustre_lnet_show_numa_range(seq_no ? seq_no->cy_valueint + : -1, + show_rc, err_rc); - return lustre_lnet_show_numa_range(seq_no ? seq_no->cy_valueint : -1, - show_rc, err_rc); + return rc; } struct lookup_cmd_hdlr_tbl { @@ -3076,7 +3259,7 @@ static struct lookup_cmd_hdlr_tbl lookup_config_tbl[] = { { .name = "peer", .cb = handle_yaml_config_peer }, { .name = "routing", .cb = handle_yaml_config_routing }, { .name = "buffers", .cb = handle_yaml_config_buffers }, - { .name = "numa", .cb = handle_yaml_config_numa }, + { .name = "global", .cb = handle_yaml_config_global_settings}, { .name = NULL } }; static struct lookup_cmd_hdlr_tbl lookup_del_tbl[] = { @@ -3084,7 +3267,7 @@ static struct lookup_cmd_hdlr_tbl lookup_del_tbl[] = { { .name = "net", .cb = handle_yaml_del_ni }, { .name = "peer", .cb = handle_yaml_del_peer }, { .name = "routing", .cb = handle_yaml_del_routing }, - { .name = "numa", .cb = handle_yaml_del_numa }, + { .name = "global", .cb = handle_yaml_del_global_settings}, { .name = NULL } }; static struct lookup_cmd_hdlr_tbl lookup_show_tbl[] = { @@ -3094,7 +3277,7 @@ static struct lookup_cmd_hdlr_tbl lookup_show_tbl[] = { { .name = "routing", .cb = handle_yaml_show_routing }, { .name = "peer", .cb = handle_yaml_show_peers }, { .name = "statistics", .cb = handle_yaml_show_stats }, - { .name = "numa", .cb = handle_yaml_show_numa }, + { .name = "global", .cb = handle_yaml_show_global_settings}, { .name = NULL } }; static cmd_handler_t lookup_fn(char *key, diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index b843b3e..8e69ebb 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -217,6 +217,30 @@ int lustre_lnet_show_numa_range(int seq_no, struct cYAML **show_rc, struct cYAML **err_rc); /* + * lustre_lnet_config_max_intf + * Sets the maximum number of interfaces per node. this tunable is + * primarily useful for sanity checks prior to allocating memory. + * + * max - maximum value to configure + * seq_no - sequence number of the request + * err_rc - [OUT] struct cYAML tree describing the error. Freed by + * caller + */ +int lustre_lnet_config_max_intf(int max, int seq_no, struct cYAML **err_rc); + +/* + * lustre_lnet_show_max_intf + * show current maximum interface setting + * + * seq_no - sequence number of the request + * show_rc - [OUT] struct cYAML tree containing NUMA range info + * err_rc - [OUT] struct cYAML tree describing the error. Freed by + * caller + */ +int lustre_lnet_show_max_intf(int seq_no, struct cYAML **show_rc, + struct cYAML **err_rc); + +/* * lustre_lnet_config_buffers * Send down an IOCTL to configure routing buffer sizes. A value of 0 means * default that particular buffer to default size. A value of -1 means diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index c2022f1..efd58dd 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -48,13 +48,14 @@ static int jt_show_net(int argc, char **argv); static int jt_show_routing(int argc, char **argv); static int jt_show_stats(int argc, char **argv); static int jt_show_peer(int argc, char **argv); -static int jt_show_numa(int argc, char **argv); +static int jt_show_global(int argc, char **argv); static int jt_set_tiny(int argc, char **argv); static int jt_set_small(int argc, char **argv); static int jt_set_large(int argc, char **argv); static int jt_set_numa(int argc, char **argv); static int jt_add_peer_nid(int argc, char **argv); static int jt_del_peer_nid(int argc, char **argv); +static int jt_set_max_intf(int argc, char **argv); /*static int jt_show_peer(int argc, char **argv);*/ static int lnetctl_list_commands(int argc, char **argv); @@ -112,8 +113,8 @@ command_t stats_cmds[] = { { 0, 0, 0, NULL } }; -command_t numa_cmds[] = { - {"show", jt_show_numa, 0, "show NUMA range\n"}, +command_t global_cmds[] = { + {"show", jt_show_global, 0, "show global variables\n"}, { 0, 0, 0, NULL } }; @@ -129,6 +130,9 @@ command_t set_cmds[] = { "\t1 - enable routing\n"}, {"numa_range", jt_set_numa, 0, "set NUMA range for NI selection\n" "\tVALUE must be at least 0\n"}, + {"max_interfaces", jt_set_max_intf, 0, "set the default value for " + "max interfaces\n" + "\tValue must be greater than 16\n"}, { 0, 0, 0, NULL } }; @@ -210,6 +214,33 @@ static int handle_help(const command_t *cmd_list, const char *cmd, return rc; } +static int jt_set_max_intf(int argc, char **argv) +{ + long int value; + int rc; + struct cYAML *err_rc = NULL; + + if (handle_help(set_cmds, "set", "max_interfaces", argc, argv) == 0) + return 0; + + rc = parse_long(argv[1], &value); + if (rc != 0) { + cYAML_build_error(-1, -1, "parser", "set", + "cannot parse max_interfaces value", &err_rc); + cYAML_print_tree2file(stderr, err_rc); + cYAML_free_tree(err_rc); + return -1; + } + + rc = lustre_lnet_config_max_intf(value, -1, &err_rc); + if (rc != LUSTRE_CFG_RC_NO_ERR) + cYAML_print_tree2file(stderr, err_rc); + + cYAML_free_tree(err_rc); + + return rc; +} + static int jt_set_numa(int argc, char **argv) { long int value; @@ -819,21 +850,30 @@ static int jt_show_stats(int argc, char **argv) return rc; } -static int jt_show_numa(int argc, char **argv) +static int jt_show_global(int argc, char **argv) { int rc; struct cYAML *show_rc = NULL, *err_rc = NULL; - if (handle_help(numa_cmds, "numa", "show", argc, argv) == 0) + if (handle_help(global_cmds, "global", "show", argc, argv) == 0) return 0; rc = lustre_lnet_show_numa_range(-1, &show_rc, &err_rc); + if (rc != LUSTRE_CFG_RC_NO_ERR) { + cYAML_print_tree2file(stderr, err_rc); + goto out; + } - if (rc != LUSTRE_CFG_RC_NO_ERR) + rc = lustre_lnet_show_max_intf(-1, &show_rc, &err_rc); + if (rc != LUSTRE_CFG_RC_NO_ERR) { cYAML_print_tree2file(stderr, err_rc); - else if (show_rc) + goto out; + } + + if (show_rc) cYAML_print_tree(show_rc); +out: cYAML_free_tree(err_rc); cYAML_free_tree(show_rc); @@ -900,16 +940,16 @@ static inline int jt_stats(int argc, char **argv) return Parser_execarg(argc - 1, &argv[1], stats_cmds); } -static inline int jt_numa(int argc, char **argv) +static inline int jt_global(int argc, char **argv) { if (argc < 2) return CMD_HELP; if (argc == 2 && - handle_help(numa_cmds, "numa", NULL, argc, argv) == 0) + handle_help(global_cmds, "global", NULL, argc, argv) == 0) return 0; - return Parser_execarg(argc - 1, &argv[1], numa_cmds); + return Parser_execarg(argc - 1, &argv[1], global_cmds); } static inline int jt_peers(int argc, char **argv) @@ -1066,6 +1106,12 @@ static int jt_export(int argc, char **argv) cYAML_free_tree(err_rc); } + rc = lustre_lnet_show_max_intf(-1, &show_rc, &err_rc); + if (rc != LUSTRE_CFG_RC_NO_ERR) { + cYAML_print_tree2file(stderr, err_rc); + cYAML_free_tree(err_rc); + } + if (show_rc != NULL) { cYAML_print_tree2file(f, show_rc); cYAML_free_tree(show_rc); @@ -1244,12 +1290,12 @@ command_t list[] = { {"net", jt_net, 0, "net {add | del | show | help}"}, {"routing", jt_routing, 0, "routing {show | help}"}, {"set", jt_set, 0, "set {tiny_buffers | small_buffers | large_buffers" - " | routing}"}, + " | routing | numa_range | max_interfaces}"}, {"import", jt_import, 0, "import {--add | --del | --show | " "--help} FILE.yaml"}, {"export", jt_export, 0, "export {--help} FILE.yaml"}, {"stats", jt_stats, 0, "stats {show | help}"}, - {"numa", jt_numa, 0, "numa {show | help}"}, + {"global", jt_global, 0, "global {show | help}"}, {"peer", jt_peers, 0, "peer {add | del | show | help}"}, {"help", Parser_help, 0, "help"}, {"exit", Parser_quit, 0, "quit"}, -- 1.8.3.1