From 86ba46c24430f67bbe0d2768f67ed0c894b2ece1 Mon Sep 17 00:00:00 2001 From: James Simmons Date: Sat, 17 Sep 2022 16:19:48 -0400 Subject: [PATCH] LU-9680 obdclass: user netlink to collect devices information Our utilities can report to users a device list with various bits of data using the debugfs file 'devices'. This debugfs file is only by default available to root which prevents regular users from collecting information. Enable non-root users to collect the same information for lctl dl using netlink. The advantage of using netlink is that it also removes the 8K ioctl limit. Add the ability to present this data in YAML format as well. Change-Id: I5e6378765bd2f4c415cf29b2bc54adf0e54f308b Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/31618 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Neil Brown Reviewed-by: Oleg Drokin --- lnet/utils/lnetconfig/liblnetconfig_netlink.c | 47 +- lustre/doc/lctl.8 | 10 +- lustre/include/lustre_kernelcomm.h | 37 +- .../include/uapi/linux/lustre/lustre_kernelcomm.h | 18 + lustre/obdclass/class_obd.c | 12 +- lustre/obdclass/kernelcomm.c | 279 +++++++++- lustre/tests/sanity.sh | 24 + lustre/utils/Makefile.am | 5 +- lustre/utils/lctl.c | 4 +- lustre/utils/lustre_cfg.c | 570 ++++++++++++++++++++- lustre/utils/obd.c | 187 +------ lustre/utils/obdctl.h | 2 +- 12 files changed, 980 insertions(+), 215 deletions(-) diff --git a/lnet/utils/lnetconfig/liblnetconfig_netlink.c b/lnet/utils/lnetconfig/liblnetconfig_netlink.c index 5c59ec6..802a294 100644 --- a/lnet/utils/lnetconfig/liblnetconfig_netlink.c +++ b/lnet/utils/lnetconfig/liblnetconfig_netlink.c @@ -1082,7 +1082,7 @@ static enum lnet_nl_key_format yaml_format_type(yaml_emitter_t *emitter, { unsigned int indent = *offset, new_indent = 0; enum lnet_nl_key_format fmt = 0; - char *tmp; + char *tmp, *flow; new_indent = indent_level(line); if (new_indent < indent) { @@ -1102,10 +1102,16 @@ static enum lnet_nl_key_format yaml_format_type(yaml_emitter_t *emitter, else fmt |= LNKF_MAPPING; - if (strchr(tmp, '{') || strchr(tmp, '[')) { + flow = strchr(line + new_indent, '{'); + if (!flow) + flow = strchr(line + new_indent, '['); + if (flow) { + if (flow < tmp) + fmt &= ~LNKF_MAPPING; fmt |= LNKF_FLOW; } else if (strchr(tmp, '}') || strchr(tmp, ']')) { - fmt &= ~LNKF_MAPPING; + if (strchr(tmp, ']')) + fmt &= ~LNKF_MAPPING; fmt |= LNKF_FLOW; } @@ -1141,10 +1147,13 @@ static int yaml_fill_scalar_data(struct nl_msg *msg, goto nla_put_failure; } - if (fmt & LNKF_MAPPING && sep && strlen(sep)) { + if (fmt & LNKF_MAPPING && sep) { while (isspace(*sep)) ++sep; + if (!strlen(sep)) + goto nla_put_failure; + if (strspn(sep, "-0123456789") == strlen(sep)) { num = strtoll(sep, NULL, 0); NLA_PUT_S64(msg, LN_SCALAR_ATTR_INT_VALUE, num); @@ -1208,13 +1217,17 @@ static int yaml_create_nested_list(struct yaml_netlink_output *out, goto nla_put_failure; } + fmt &= ~LNKF_FLOW; while ((line = strsep(hdr, ",")) != NULL) { while (!isalnum(line[0])) line++; - /* Flow can be splt across lines by libyaml library. - * This is executed only once. - */ + /* Flow can be splt across lines by libyaml library. */ + if (strchr(line, ',')) { + *hdr = line; + continue; + } + tmp = strchr(line, '}'); if (!tmp) tmp = strchr(line, ']'); @@ -1228,9 +1241,10 @@ static int yaml_create_nested_list(struct yaml_netlink_output *out, goto nla_put_failure; /* Move to next YAML line */ - tmp = strstr(*entry, line); - if (tmp && strcmp(tmp, line) == 0) + if (format) { strsep(entry, "\n"); + break; + } } if (!format) { @@ -1258,7 +1272,6 @@ have_next_line: fmt); if (rc < 0) goto nla_put_failure; - if (line) goto have_next_line; } else { @@ -1269,6 +1282,13 @@ have_next_line: } } while (strcmp(*entry, "")); + if (line && line[0] == '-' && !*indent) { + line[0] = ' '; + *indent = 2; + goto have_next_line; + } + if (*entry && !strlen(*entry)) + line = NULL; /* strsep in the above loop moves entry to a value pass the * end of the nested list. So to avoid losing this value we * replace hdr with line. @@ -1360,8 +1380,8 @@ already_have_line: * simple key */ if (extra) { - line = ++tmp; - indent = 1; + *tmp = ' '; + line = tmp; goto already_have_line; } } else { @@ -1384,6 +1404,9 @@ already_have_line: nlmsg_free(msg); goto nla_put_failure; } + + if (line[0] != '-') + indent = 2; } fmt = yaml_format_type(out->emitter, line, &indent); diff --git a/lustre/doc/lctl.8 b/lustre/doc/lctl.8 index 64d147c..86a6c6c 100644 --- a/lustre/doc/lctl.8 +++ b/lustre/doc/lctl.8 @@ -100,10 +100,14 @@ Print the complete routing table. .BI device " " This will select the specified OBD device. All other commands depend on the device being set. .TP -.BI device_list -Show all the local Lustre OBDs. AKA +.BI device_list " [--target|-t] [--yaml|-y] " +Show all the local Lustre OBDs. .B dl -.PP +is the same command. The +.B -t +option additonally will report the server's export NID linked to the local OBD. The +.B -y +option will display the OBD device information in YAML format. .SS Device Operations .TP .BI conf_param " [-d] .=" diff --git a/lustre/include/lustre_kernelcomm.h b/lustre/include/lustre_kernelcomm.h index 4af88af..b6f5bb6 100644 --- a/lustre/include/lustre_kernelcomm.h +++ b/lustre/include/lustre_kernelcomm.h @@ -40,11 +40,46 @@ /* For declarations shared with userspace */ #include +/** + * enum lustre_device_attrs - Lustre general top-level netlink + * attributes that describe lustre + * 'devices'. These values are used + * to piece togther messages for + * sending and receiving. + * + * @LUSTRE_DEVICE_ATTR_UNSPEC: unspecified attribute to catch errors + * + * @LUSTRE_DEVICE_ATTR_HDR: Netlink group this data is for + * (NLA_NUL_STRING) + * @LUSTRE_DEVICE_ATTR_INDEX: device number used as an index (NLA_U16) + * @LUSTRE_DEVICE_ATTR_STATUS: status of the device (NLA_STRING) + * @LUSTRE_DEVICE_ATTR_CLASS: class the device belongs to (NLA_STRING) + * @LUSTRE_DEVICE_ATTR_NAME: name of the device (NLA_STRING) + * @LUSTRE_DEVICE_ATTR_UUID: UUID of the device (NLA_STRING) + * @LUSTRE_DEVICE_ATTR_REFCOUNT: refcount of the device (NLA_U32) + */ +enum lustre_device_attrs { + LUSTRE_DEVICE_ATTR_UNSPEC = 0, + + LUSTRE_DEVICE_ATTR_HDR, + LUSTRE_DEVICE_ATTR_INDEX, + LUSTRE_DEVICE_ATTR_STATUS, + LUSTRE_DEVICE_ATTR_CLASS, + LUSTRE_DEVICE_ATTR_NAME, + LUSTRE_DEVICE_ATTR_UUID, + LUSTRE_DEVICE_ATTR_REFCOUNT, + + __LUSTRE_DEVICE_ATTR_MAX_PLUS_ONE +}; + +#define LUSTRE_DEVICE_ATTR_MAX (__LUSTRE_DEVICE_ATTR_MAX_PLUS_ONE - 1) + /* prototype for callback function on kuc groups */ typedef int (*libcfs_kkuc_cb_t)(void *data, void *cb_arg); /* Kernel methods */ -void libcfs_kkuc_init(void); +int libcfs_kkuc_init(void); +void libcfs_kkuc_fini(void); int libcfs_kkuc_msg_put(struct file *fp, void *payload); int libcfs_kkuc_group_put(const struct obd_uuid *uuid, int group, void *data); int libcfs_kkuc_group_add(struct file *fp, const struct obd_uuid *uuid, int uid, diff --git a/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h b/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h index 26819ff..27de4b2 100644 --- a/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h +++ b/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h @@ -38,6 +38,24 @@ #include +#define LUSTRE_GENL_NAME "lustre" +#define LUSTRE_GENL_VERSION 0x1 + +/* + * enum lustre_commands - Supported Lustre Netlink commands + * + * @LUSTRE_CMD_UNSPEC: unspecified command to catch errors + * @LUSTRE_CMD_DEVICES: command to manage the Lustre devices + */ +enum lustre_commands { + LUSTRE_CMD_UNSPEC = 0, + LUSTRE_CMD_DEVICES = 1, + + __LUSTRE_CMD_MAX_PLUS_ONE +}; + +#define LUSTRE_CMD_MAX (__LUSTRE_CMD_MAX_PLUS_ONE - 1) + /* KUC message header. * All current and future KUC messages should use this header. * To avoid having to include Lustre headers from libcfs, define this here. diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c index f949ac4..f323d15 100644 --- a/lustre/obdclass/class_obd.c +++ b/lustre/obdclass/class_obd.c @@ -683,8 +683,6 @@ static int __init obdclass_init(void) register_oom_notifier(&obdclass_oom); - libcfs_kkuc_init(); - err = obd_init_checks(); if (err) return err; @@ -702,10 +700,14 @@ static int __init obdclass_init(void) LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES, "memused"); #endif - err = obd_zombie_impexp_init(); + err = libcfs_kkuc_init(); if (err) goto cleanup_obd_memory; + err = obd_zombie_impexp_init(); + if (err) + goto cleanup_kkuc; + err = class_handle_init(); if (err) goto cleanup_zombie_impexp; @@ -796,6 +798,9 @@ cleanup_class_handle: cleanup_zombie_impexp: obd_zombie_impexp_stop(); +cleanup_kkuc: + libcfs_kkuc_fini(); + cleanup_obd_memory: #ifdef CONFIG_PROC_FS lprocfs_free_stats(&obd_memory); @@ -856,6 +861,7 @@ static void __exit obdclass_exit(void) class_handle_cleanup(); class_del_uuid(NULL); /* Delete all UUIDs. */ obd_zombie_impexp_stop(); + libcfs_kkuc_fini(); #ifdef CONFIG_PROC_FS memory_leaked = obd_memory_sum(); diff --git a/lustre/obdclass/kernelcomm.c b/lustre/obdclass/kernelcomm.c index 7afb948..98c21e8 100644 --- a/lustre/obdclass/kernelcomm.c +++ b/lustre/obdclass/kernelcomm.c @@ -37,10 +37,280 @@ #define DEBUG_SUBSYSTEM S_CLASS #include +#include +#include +#include +#include +#include #include #include +static struct genl_family lustre_family; + +static struct ln_key_list device_list = { + .lkl_maxattr = LUSTRE_DEVICE_ATTR_MAX, + .lkl_list = { + [LUSTRE_DEVICE_ATTR_HDR] = { + .lkp_value = "devices", + .lkp_key_format = LNKF_SEQUENCE | LNKF_MAPPING, + .lkp_data_type = NLA_NUL_STRING, + }, + [LUSTRE_DEVICE_ATTR_INDEX] = { + .lkp_value = "index", + .lkp_data_type = NLA_U16 + }, + [LUSTRE_DEVICE_ATTR_STATUS] = { + .lkp_value = "status", + .lkp_data_type = NLA_STRING + }, + [LUSTRE_DEVICE_ATTR_CLASS] = { + .lkp_value = "type", + .lkp_data_type = NLA_STRING + }, + [LUSTRE_DEVICE_ATTR_NAME] = { + .lkp_value = "name", + .lkp_data_type = NLA_STRING + }, + [LUSTRE_DEVICE_ATTR_UUID] = { + .lkp_value = "uuid", + .lkp_data_type = NLA_STRING + }, + [LUSTRE_DEVICE_ATTR_REFCOUNT] = { + .lkp_value = "refcount", + .lkp_data_type = NLA_U32 + }, + }, +}; + +struct genl_dev_list { + struct obd_device *gdl_target; + unsigned int gdl_start; +}; + +static inline struct genl_dev_list * +device_dump_ctx(struct netlink_callback *cb) +{ + return (struct genl_dev_list *)cb->args[0]; +} + +/* generic ->start() handler for GET requests */ +static int lustre_device_list_start(struct netlink_callback *cb) +{ + struct genlmsghdr *gnlh = nlmsg_data(cb->nlh); +#ifdef HAVE_NL_PARSE_WITH_EXT_ACK + struct netlink_ext_ack *extack = NULL; +#endif + struct genl_dev_list *glist; + int msg_len, rc = 0; + +#ifdef HAVE_NL_DUMP_WITH_EXT_ACK + extack = cb->extack; +#endif + OBD_ALLOC(glist, sizeof(*glist)); + if (!glist) + return -ENOMEM; + + cb->args[0] = (long)glist; + glist->gdl_target = NULL; + glist->gdl_start = 0; + + msg_len = genlmsg_len(gnlh); + if (msg_len > 0) { + struct nlattr *params = genlmsg_data(gnlh); + struct nlattr *dev; + int rem; + + nla_for_each_attr(dev, params, msg_len, rem) { + struct nlattr *prop; + int rem2; + + nla_for_each_nested(prop, dev, rem2) { + char name[MAX_OBD_NAME]; + struct obd_device *obd; + + if (nla_type(prop) != LN_SCALAR_ATTR_VALUE || + nla_strcmp(prop, "name") != 0) + continue; + + prop = nla_next(prop, &rem2); + if (nla_type(prop) != LN_SCALAR_ATTR_VALUE) + GOTO(report_err, rc = -EINVAL); + + rc = nla_strscpy(name, prop, sizeof(name)); + if (rc < 0) + GOTO(report_err, rc); + rc = 0; + + obd = class_name2obd(name); + if (obd) + glist->gdl_target = obd; + } + } + if (!glist->gdl_target) { + NL_SET_ERR_MSG(extack, "No devices found"); + rc = -ENOENT; + } + } +report_err: + if (rc < 0) { + OBD_FREE(glist, sizeof(*glist)); + cb->args[0] = 0; + } + return rc; +} + +static int lustre_device_list_dump(struct sk_buff *msg, + struct netlink_callback *cb) +{ + struct genl_dev_list *glist = device_dump_ctx(cb); + struct obd_device *filter = glist->gdl_target; +#ifdef HAVE_NL_PARSE_WITH_EXT_ACK + struct netlink_ext_ack *extack = NULL; +#endif + int portid = NETLINK_CB(cb->skb).portid; + int seq = cb->nlh->nlmsg_seq; + int idx, rc = 0; + +#ifdef HAVE_NL_DUMP_WITH_EXT_ACK + extack = cb->extack; +#endif + if (glist->gdl_start == 0) { + const struct ln_key_list *all[] = { + &device_list, NULL + }; + + rc = lnet_genl_send_scalar_list(msg, portid, seq, + &lustre_family, + NLM_F_CREATE | NLM_F_MULTI, + LUSTRE_CMD_DEVICES, all); + if (rc < 0) { + NL_SET_ERR_MSG(extack, "failed to send key table"); + return rc; + } + } + + for (idx = glist->gdl_start; idx < class_devno_max(); idx++) { + struct obd_device *obd; + const char *status; + void *hdr; + + obd = class_num2obd(idx); + if (!obd) + continue; + + if (filter && filter != obd) + continue; + + hdr = genlmsg_put(msg, portid, seq, &lustre_family, + NLM_F_MULTI, LUSTRE_CMD_DEVICES); + if (!hdr) { + NL_SET_ERR_MSG(extack, "failed to send values"); + genlmsg_cancel(msg, hdr); + rc = -EMSGSIZE; + break; + } + + if (idx == 0) + nla_put_string(msg, LUSTRE_DEVICE_ATTR_HDR, ""); + + nla_put_u16(msg, LUSTRE_DEVICE_ATTR_INDEX, obd->obd_minor); + + /* Collect only the index value for a single obd */ + if (filter) { + genlmsg_end(msg, hdr); + idx++; + break; + } + + if (obd->obd_stopping) + status = "ST"; + else if (obd->obd_inactive) + status = "IN"; + else if (obd->obd_set_up) + status = "UP"; + else if (obd->obd_attached) + status = "AT"; + else + status = "--"; + + nla_put_string(msg, LUSTRE_DEVICE_ATTR_STATUS, status); + + nla_put_string(msg, LUSTRE_DEVICE_ATTR_CLASS, + obd->obd_type->typ_name); + + nla_put_string(msg, LUSTRE_DEVICE_ATTR_NAME, + obd->obd_name); + + nla_put_string(msg, LUSTRE_DEVICE_ATTR_UUID, + obd->obd_uuid.uuid); + + nla_put_u32(msg, LUSTRE_DEVICE_ATTR_REFCOUNT, + atomic_read(&obd->obd_refcount)); + + genlmsg_end(msg, hdr); + } + + glist->gdl_start = idx; + rc = lnet_nl_send_error(cb->skb, portid, seq, rc); + return rc < 0 ? rc : msg->len; +} + +#ifndef HAVE_NETLINK_CALLBACK_START +int lustre_old_device_list_dump(struct sk_buff *msg, + struct netlink_callback *cb) +{ + if (!cb->args[0]) { + int rc = lustre_device_list_start(cb); + + if (rc < 0) + return rc; + } + + return lustre_device_list_dump(msg, cb); +} +#endif + +int lustre_device_done(struct netlink_callback *cb) +{ + struct genl_dev_list *glist; + + glist = device_dump_ctx(cb); + if (glist) { + OBD_FREE(glist, sizeof(*glist)); + cb->args[0] = 0; + } + + return 0; +} + +static const struct genl_multicast_group lustre_mcast_grps[] = { + { .name = "devices", }, +}; + +static const struct genl_ops lustre_genl_ops[] = { + { + .cmd = LUSTRE_CMD_DEVICES, +#ifdef HAVE_NETLINK_CALLBACK_START + .start = lustre_device_list_start, + .dumpit = lustre_device_list_dump, +#else + .dumpit = lustre_old_device_list_dump, +#endif + .done = lustre_device_done, + }, +}; + +static struct genl_family lustre_family = { + .name = LUSTRE_GENL_NAME, + .version = LUSTRE_GENL_VERSION, + .module = THIS_MODULE, + .ops = lustre_genl_ops, + .n_ops = ARRAY_SIZE(lustre_genl_ops), + .mcgrps = lustre_mcast_grps, + .n_mcgrps = ARRAY_SIZE(lustre_mcast_grps), +}; + /** * libcfs_kkuc_msg_put - send an message from kernel to userspace * @param fp to send the message to @@ -101,12 +371,19 @@ static inline bool libcfs_kkuc_group_is_valid(int group) return 0 <= group && group < ARRAY_SIZE(kkuc_groups); } -void libcfs_kkuc_init(void) +int libcfs_kkuc_init(void) { int group; for (group = 0; group < ARRAY_SIZE(kkuc_groups); group++) INIT_LIST_HEAD(&kkuc_groups[group]); + + return genl_register_family(&lustre_family); +} + +void libcfs_kkuc_fini(void) +{ + genl_unregister_family(&lustre_family); } /** Add a receiver to a broadcast group diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index e649462..2c1711b 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -11976,6 +11976,30 @@ test_104c() { } run_test 104c "Verify df vs lfs_df stays same after recordsize change" +test_104d() { + (( $RUNAS_ID != $UID )) || + skip_env "RUNAS_ID = UID = $UID -- skipping" + + (( $CLIENT_VERSION >= $(version_code 2.15.51) )) || + skip "lustre version doesn't support lctl dl with non-root" + + # debugfs only allows root users to access files, so the + # previous move of the "devices" file to debugfs broke + # "lctl dl" for non-root users. The LU-9680 Netlink + # interface again allows non-root users to list devices. + [ "$($RUNAS $LCTL dl | wc -l)" -ge 3 ] || + error "lctl dl doesn't work for non root" + + ost_count="$($RUNAS $LCTL dl | grep $FSNAME-OST* | wc -l)" + [ "$ost_count" -eq $OSTCOUNT ] || + error "lctl dl reports wrong number of OST devices" + + mdt_count="$($RUNAS $LCTL dl | grep $FSNAME-MDT* | wc -l)" + [ "$mdt_count" -eq $MDSCOUNT ] || + error "lctl dl reports wrong number of MDT devices" +} +run_test 104d "$RUNAS lctl dl test" + test_105a() { # doesn't work on 2.4 kernels touch $DIR/$tfile diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index aa517b9..55863ef 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -71,8 +71,9 @@ lctl_SOURCES = portals.c debug.c obd.c lustre_cfg.c lctl.c obdctl.h if SERVER lctl_SOURCES += lustre_lfsck.c lsnapshot.c endif -lctl_LDADD := liblustreapi.la $(PTHREAD_LIBS) -lyaml -lctl_DEPENDENCIES := liblustreapi.la +lctl_CFLAGS := $(AM_CFLAGS) $(LIBNL3_CFLAGS) -I $(top_builddir)/lnet/utils +lctl_LDADD := liblustreapi.la $(PTHREAD_LIBS) $(LIBNL3_LIBS) -lyaml +lctl_LDADD += $(top_builddir)/lnet/utils/lnetconfig/liblnetconfig.la lfs_SOURCES = lfs.c lfs_project.c lfs_project.h lfs_CFLAGS := -fPIC $(AM_CFLAGS) -I $(top_builddir)/lnet/utils diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 1765661..5a6db60 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -183,9 +183,9 @@ command_t cmdlist[] = { "set current device to , same as 'device'\n" "usage: cfg_device "}, {"device_list", jt_obd_list, 0, "show all devices\n" - "usage: device_list"}, + "usage: device_list [--target|-t] [--yaml|-y]"}, {"dl", jt_obd_list, 0, "show all devices, same as 'device_list'\n" - "usage: dl [-t]"}, + "usage: dl [--target|-t] [--yaml|-y]"}, /* Device operations */ {"==== obd device operations ====", NULL, 0, "device operations"}, diff --git a/lustre/utils/lustre_cfg.c b/lustre/utils/lustre_cfg.c index 8394f85..0b2a603 100644 --- a/lustre/utils/lustre_cfg.c +++ b/lustre/utils/lustre_cfg.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -60,6 +61,10 @@ #include #include +#include +#include +#include "lustreapi_internal.h" + #include #include #include @@ -442,6 +447,8 @@ struct param_opts { unsigned int po_delete:1; unsigned int po_only_dir:1; unsigned int po_file:1; + unsigned int po_yaml:1; + unsigned int po_detail:1; }; int lcfg_setparam_perm(char *func, char *buf) @@ -519,7 +526,7 @@ static int jt_lcfg_setparam_perm(int argc, char **argv, if (end_pos) { *(++end_pos) = '\0'; } else if (buf[len - 1] != '=') { - buf = malloc(len + 1); + buf = malloc(len + 2); if (buf == NULL) return -ENOMEM; sprintf(buf, "%s=", argv[i]); @@ -882,6 +889,362 @@ write_param(const char *path, const char *param_name, struct param_opts *popt, return rc; } +void print_obd_line(char *s) +{ + const char *param = "osc/%s/ost_conn_uuid"; + char obd_name[MAX_OBD_NAME]; + char buf[MAX_OBD_NAME]; + FILE *fp = NULL; + glob_t path; + char *ptr; +retry: + /* obd device type is the first 3 characters of param name */ + snprintf(buf, sizeof(buf), " %%*d %%*s %.3s %%%zus %%*s %%*d ", + param, sizeof(obd_name) - 1); + if (sscanf(s, buf, obd_name) == 0) + goto try_mdc; + if (cfs_get_param_paths(&path, param, obd_name) != 0) + goto try_mdc; + fp = fopen(path.gl_pathv[0], "r"); + if (!fp) { + /* need to free path data before retry */ + cfs_free_param_data(&path); +try_mdc: + if (param[0] == 'o') { /* failed with osc, try mdc */ + param = "mdc/%s/mds_conn_uuid"; + goto retry; + } + buf[0] = '\0'; + goto fail_print; + } + + /* should not ignore fgets(3)'s return value */ + if (!fgets(buf, sizeof(buf), fp)) { + fprintf(stderr, "reading from %s: %s", buf, strerror(errno)); + goto fail_close; + } + +fail_close: + fclose(fp); + cfs_free_param_data(&path); + + /* trim trailing newlines */ + ptr = strrchr(buf, '\n'); + if (ptr) + *ptr = '\0'; +fail_print: + ptr = strrchr(s, '\n'); + if (ptr) + *ptr = '\0'; + printf("%s%s%s\n", s, buf[0] ? " " : "", buf); +} + +static int print_out_devices(yaml_parser_t *reply, struct param_opts *popt) +{ + char buf[MAX_OBD_NAME], *tmp = NULL; + size_t buf_len = sizeof(buf); + yaml_event_t event; + bool done = false; + int rc; + + bzero(buf, sizeof(buf)); + + while (!done) { + rc = yaml_parser_parse(reply, &event); + if (rc == 0) + break; + + if (event.type == YAML_MAPPING_START_EVENT) { + size_t len = strlen(buf); + + if (len > 0) { + /* eat last white space */ + buf[len - 1] = '\0'; + if (popt->po_detail) + print_obd_line(buf); + else + printf("%s\n", buf); + } + bzero(buf, sizeof(buf)); + tmp = buf; + } + + if (event.type == YAML_SCALAR_EVENT) { + char *value = (char *)event.data.scalar.value; + + if (strcmp(value, "index") == 0) { + yaml_event_delete(&event); + rc = yaml_parser_parse(reply, &event); + if (rc == 0) + break; + + value = (char *)event.data.scalar.value; + + snprintf(tmp, buf_len, "%3s ", value); + buf_len -= 4; + tmp += 4; + } + + if (strcmp(value, "status") == 0 || + strcmp(value, "type") == 0 || + strcmp(value, "name") == 0 || + strcmp(value, "uuid") == 0 || + strcmp(value, "refcount") == 0) { + yaml_event_delete(&event); + rc = yaml_parser_parse(reply, &event); + if (rc == 0) + break; + + value = (char *)event.data.scalar.value; + + snprintf(tmp, buf_len, "%s ", value); + buf_len -= strlen(value) + 1; + tmp += strlen(value) + 1; + } + } + + done = (event.type == YAML_DOCUMENT_END_EVENT); + if (done) { + size_t len = strlen(buf); + + if (len > 0) { + /* eat last white space */ + buf[len - 1] = '\0'; + if (popt->po_detail) + print_obd_line(buf); + else + printf("%s\n", buf); + } + bzero(buf, sizeof(buf)); + tmp = buf; + } + yaml_event_delete(&event); + } + + return rc; +} + +int lcfg_param_get_yaml(yaml_parser_t *reply, struct nl_sock *sk, char *pattern) +{ + char source[MAX_OBD_NAME], group[GENL_NAMSIZ + 1]; + int version = LUSTRE_GENL_VERSION; + char *family = "lustre", *tmp; + yaml_emitter_t request; + yaml_event_t event; + int cmd = 0; + int rc; + + bzero(source, sizeof(source)); + tmp = strrchr(pattern, '/'); + if (tmp) { + size_t len = tmp - pattern; + + strncpy(group, tmp + 1, GENL_NAMSIZ); + strncpy(source, pattern, len); + + /* replace '/' with '.' to match conf_param and sysctl */ + for (tmp = strchr(pattern, '/'); tmp != NULL; + tmp = strchr(tmp, '/')) + *tmp = '.'; + } else { + strncpy(group, pattern, GENL_NAMSIZ); + } + + if (strcmp(group, "devices") == 0) + cmd = LUSTRE_CMD_DEVICES; + + if (!cmd) + return -EOPNOTSUPP; + + /* Setup parser to recieve Netlink packets */ + rc = yaml_parser_initialize(reply); + if (rc == 0) + return -EOPNOTSUPP; + + rc = yaml_parser_set_input_netlink(reply, sk, false); + if (rc == 0) + return -EOPNOTSUPP; + + /* Create Netlink emitter to send request to kernel */ + yaml_emitter_initialize(&request); + rc = yaml_emitter_set_output_netlink(&request, sk, + family, version, + cmd, NLM_F_DUMP); + if (rc == 0) + goto error; + + yaml_emitter_open(&request); + + yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_mapping_start_event_initialize(&event, NULL, + (yaml_char_t *)YAML_MAP_TAG, + 1, YAML_ANY_MAPPING_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)group, + strlen(group), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + if (source[0]) { + const char *key = "name"; + + /* Now fill in 'path' filter */ + yaml_sequence_start_event_initialize(&event, NULL, + (yaml_char_t *)YAML_SEQ_TAG, + 1, YAML_ANY_SEQUENCE_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_mapping_start_event_initialize(&event, NULL, + (yaml_char_t *)YAML_MAP_TAG, + 1, YAML_ANY_MAPPING_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)key, strlen(key), + 1, 0, YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)source, + strlen(source), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_mapping_end_event_initialize(&event); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_sequence_end_event_initialize(&event); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + } else { + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"", + strlen(""), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + } + yaml_mapping_end_event_initialize(&event); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_document_end_event_initialize(&event, 0); + rc = yaml_emitter_emit(&request, &event); + if (rc == 0) + goto error; + + yaml_emitter_close(&request); +error: + if (rc == 0) { + yaml_emitter_log_error(&request, stderr); + rc = -EINVAL; + } + yaml_emitter_delete(&request); + + return rc == 1 ? 0 : -EINVAL; +} + +int lcfg_getparam_yaml(char *path, struct param_opts *popt) +{ + yaml_parser_t reply; + struct nl_sock *sk; + int rc; + + sk = nl_socket_alloc(); + if (!sk) + return -ENOMEM; + + rc = lcfg_param_get_yaml(&reply, sk, path); + if (rc < 0) + return rc; + + if (popt->po_yaml) { + yaml_document_t results; + yaml_emitter_t output; + + /* load the reply results */ + rc = yaml_parser_load(&reply, &results); + if (rc == 0) { + yaml_parser_log_error(&reply, stderr, "get_param: "); + yaml_document_delete(&results); + rc = -EINVAL; + goto free_reply; + } + + /* create emitter to output results */ + rc = yaml_emitter_initialize(&output); + if (rc == 1) { + yaml_emitter_set_output_file(&output, stdout); + + rc = yaml_emitter_dump(&output, &results); + } + + yaml_document_delete(&results); + if (rc == 0) { + yaml_emitter_log_error(&output, stderr); + rc = -EINVAL; + } + yaml_emitter_delete(&output); + } else { + yaml_event_t event; + bool done = false; + + while (!done) { + rc = yaml_parser_parse(&reply, &event); + if (rc == 0) + break; + + if (event.type == YAML_SCALAR_EVENT) { + char *value = (char *)event.data.scalar.value; + + if (strcmp(value, "devices") == 0) + rc = print_out_devices(&reply, popt); + if (rc == 0) + break; + } + + done = (event.type == YAML_STREAM_END_EVENT); + yaml_event_delete(&event); + } + + if (rc == 0) { + yaml_parser_log_error(&reply, stderr, "get_param: "); + rc = -EINVAL; + } + } +free_reply: + yaml_parser_delete(&reply); + nl_socket_free(sk); + return rc == 1 ? 0 : rc; +} + /** * Perform a read, write or just a listing of a parameter * @@ -906,7 +1269,7 @@ param_display(struct param_opts *popt, char *pattern, char *value, rc = llapi_param_get_paths(pattern, &paths); if (rc != 0) { rc = -errno; - if (!popt->po_recursive) { + if (!popt->po_recursive && !(rc == -ENOENT && getuid() != 0)) { fprintf(stderr, "error: %s: param_path '%s': %s\n", opname, pattern, strerror(errno)); } @@ -1165,6 +1528,7 @@ static int getparam_cmdline(int argc, char **argv, struct param_opts *popt) int jt_lcfg_getparam(int argc, char **argv) { + enum parameter_operation mode; int rc = 0, index, i; struct param_opts popt; char *path; @@ -1174,6 +1538,7 @@ int jt_lcfg_getparam(int argc, char **argv) if (index < 0 || index >= argc) return CMD_HELP; + mode = popt.po_only_path ? LIST_PARAM : GET_PARAM; for (i = index; i < argc; i++) { int rc2; @@ -1188,9 +1553,12 @@ int jt_lcfg_getparam(int argc, char **argv) continue; } - rc2 = param_display(&popt, path, NULL, - popt.po_only_path ? LIST_PARAM : GET_PARAM); + rc2 = param_display(&popt, path, NULL, mode); if (rc2 < 0) { + if (mode == GET_PARAM && rc2 == -ENOENT && + getuid() != 0) + rc2 = lcfg_getparam_yaml(path, &popt); + if (rc == 0) rc = rc2; continue; @@ -1200,6 +1568,200 @@ int jt_lcfg_getparam(int argc, char **argv) return rc; } +/* get device list by netlink or debugfs */ +int jt_obd_list(int argc, char **argv) +{ + static const struct option long_opts[] = { + { .name = "target", .has_arg = no_argument, .val = 't' }, + { .name = "yaml", .has_arg = no_argument, .val = 'y' }, + { .name = NULL } + }; + struct param_opts opts; + char buf[MAX_OBD_NAME]; + struct nl_sock *sk; + glob_t path; + int rc, c; + FILE *fp; + + if (optind < argc) + return CMD_HELP; + + memset(&opts, 0, sizeof(opts)); + + while ((c = getopt_long(argc, argv, "ty", long_opts, NULL)) != -1) { + switch (c) { + case 't': + opts.po_detail = true; + break; + case 'y': + opts.po_yaml = true; + break; + default: + return CMD_HELP; + } + } + + if (optind < argc) { + optind = 1; + return CMD_HELP; + } + optind = 1; + + sk = nl_socket_alloc(); + if (!sk) + goto non_netlink; + + /* Use YAML to list all devices */ + rc = lcfg_getparam_yaml("devices", &opts); + if (rc == 0) + return 0; + +non_netlink: + if (sk) { + nl_close(sk); + nl_socket_free(sk); + } + + rc = llapi_param_get_paths("devices", &path); + if (rc < 0) + return rc; + + fp = fopen(path.gl_pathv[0], "r"); + if (!fp) { + cfs_free_param_data(&path); + return errno; + } + + while (fgets(buf, sizeof(buf), fp) != NULL) + if (opts.po_detail) + print_obd_line(buf); + else + printf("%s", buf); + + cfs_free_param_data(&path); + fclose(fp); + return 0; +} + +static int do_name2dev(char *func, char *name, int dev_id) +{ + struct obd_ioctl_data data; + char rawbuf[MAX_IOC_BUFLEN], *buf = rawbuf; + yaml_parser_t output; + yaml_event_t event; + struct nl_sock *sk; + bool done = false; + int rc = 0; + + memset(buf, 0, sizeof(rawbuf)); + scnprintf(buf, sizeof(rawbuf), "%s/devices", + name); + + sk = nl_socket_alloc(); + if (!sk) + goto ioctl; + + /* Use YAML to list all devices */ + rc = lcfg_param_get_yaml(&output, sk, buf); + if (rc < 0) { + if (rc == -EOPNOTSUPP) + goto ioctl; + return rc; + } + + while (!done) { + rc = yaml_parser_parse(&output, &event); + if (rc == 0) { + yaml_parser_log_error(&output, stdout, "lctl: "); + rc = -EINVAL; + break; + } + + if (event.type == YAML_SCALAR_EVENT) { + char *value = (char *)event.data.scalar.value; + + if (strcmp(value, "index") == 0) { + yaml_event_delete(&event); + rc = yaml_parser_parse(&output, &event); + if (rc == 1) { + value = (char *)event.data.scalar.value; + errno = 0; + rc = strtoul(value, NULL, 10); + if (errno) { + yaml_event_delete(&event); + rc = -errno; + goto ioctl; + } + return rc; + } + } + } + done = (event.type == YAML_STREAM_END_EVENT); + yaml_event_delete(&event); + } +ioctl: + if (sk) + nl_socket_free(sk); + + if (rc > 0 || rc != -EOPNOTSUPP) + return rc; + + memset(&data, 0, sizeof(data)); + data.ioc_dev = dev_id; + data.ioc_inllen1 = strlen(name) + 1; + data.ioc_inlbuf1 = name; + + memset(buf, 0, sizeof(rawbuf)); + rc = llapi_ioctl_pack(&data, &buf, sizeof(rawbuf)); + if (rc < 0) { + fprintf(stderr, "error: %s: invalid ioctl\n", jt_cmdname(func)); + return rc; + } + rc = l_ioctl(OBD_DEV_ID, OBD_IOC_NAME2DEV, buf); + if (rc < 0) + return -errno; + rc = llapi_ioctl_unpack(&data, buf, sizeof(rawbuf)); + if (rc < 0) { + fprintf(stderr, "error: %s: invalid reply\n", jt_cmdname(func)); + return rc; + } + + return data.ioc_dev; +} + +/* + * resolve a device name to a device number. + * supports a number, $name or %uuid. + */ +int parse_devname(char *func, char *name, int dev_id) +{ + int rc = 0; + int ret = -1; + + if (!name) + return ret; + + /* Test if its a pure number string */ + if (strspn(name, "0123456789") != strlen(name)) { + if (name[0] == '$' || name[0] == '%') + name++; + + rc = do_name2dev(func, name, dev_id); + if (rc >= 0) + ret = rc; + } else { + errno = 0; + ret = strtoul(name, NULL, 10); + if (errno) + rc = errno; + } + + if (rc < 0) + fprintf(stderr, "No device found for name %s: %s\n", + name, strerror(rc)); + return ret; +} + #ifdef HAVE_SERVER_SUPPORT /** * Output information about nodemaps. diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index 2e5130e..ebb1fcb 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -214,67 +214,6 @@ char *obdo_print(struct obdo *obd) #define BAD_VERBOSE (-999999999) -#define N2D_OFF 0x100 /* So we can tell between error codes and devices */ - -static int do_name2dev(char *func, char *name) -{ - struct obd_ioctl_data data; - char rawbuf[MAX_IOC_BUFLEN], *buf = rawbuf; - int rc; - - memset(&data, 0, sizeof(data)); - data.ioc_dev = cur_device; - data.ioc_inllen1 = strlen(name) + 1; - data.ioc_inlbuf1 = name; - - memset(buf, 0, sizeof(rawbuf)); - rc = llapi_ioctl_pack(&data, &buf, sizeof(rawbuf)); - if (rc < 0) { - fprintf(stderr, "error: %s: invalid ioctl\n", jt_cmdname(func)); - return -rc; - } - rc = l_ioctl(OBD_DEV_ID, OBD_IOC_NAME2DEV, buf); - if (rc < 0) - return errno; - rc = llapi_ioctl_unpack(&data, buf, sizeof(rawbuf)); - if (rc) { - fprintf(stderr, "error: %s: invalid reply\n", jt_cmdname(func)); - return rc; - } - - return data.ioc_dev + N2D_OFF; -} - -/* - * resolve a device name to a device number. - * supports a number, $name or %uuid. - */ -int parse_devname(char *func, char *name) -{ - int rc; - int ret = -1; - int try_digit; - - if (!name) - return ret; - - try_digit = isdigit(name[0]); - - if (name[0] == '$' || name[0] == '%') - name++; - - rc = do_name2dev(func, name); - if (rc >= N2D_OFF) - return rc - N2D_OFF; - - if (try_digit) - ret = strtoul(name, NULL, 0); - else - fprintf(stderr, "No device found for name %s: %s\n", - name, strerror(rc)); - return ret; -} - char *jt_cmdname(char *func) { static char buf[512]; @@ -616,7 +555,7 @@ static int do_device(char *func, char *devname) { int dev; - dev = parse_devname(func, devname); + dev = parse_devname(func, devname, cur_device); if (dev < 0) return -1; @@ -1031,130 +970,6 @@ int jt_get_version(int argc, char **argv) return 0; } -static void print_obd_line(char *s) -{ - const char *param = "osc/%s/ost_conn_uuid"; - char buf[MAX_STRING_SIZE]; - char obd_name[MAX_OBD_NAME]; - FILE *fp = NULL; - glob_t path; - char *ptr; -retry: - /* obd device type is the first 3 characters of param name */ - snprintf(buf, sizeof(buf), " %%*d %%*s %.3s %%%zus %%*s %%*d ", - param, sizeof(obd_name) - 1); - if (sscanf(s, buf, obd_name) == 0) - goto try_mdc; - if (cfs_get_param_paths(&path, param, obd_name) != 0) - goto try_mdc; - fp = fopen(path.gl_pathv[0], "r"); - if (!fp) { - /* need to free path data before retry */ - cfs_free_param_data(&path); -try_mdc: - if (param[0] == 'o') { /* failed with osc, try mdc */ - param = "mdc/%s/mds_conn_uuid"; - goto retry; - } - buf[0] = '\0'; - goto fail_print; - } - - /* should not ignore fgets(3)'s return value */ - if (!fgets(buf, sizeof(buf), fp)) { - fprintf(stderr, "reading from %s: %s", buf, strerror(errno)); - goto fail_close; - } - -fail_close: - fclose(fp); - cfs_free_param_data(&path); - - /* trim trailing newlines */ - ptr = strrchr(buf, '\n'); - if (ptr) - *ptr = '\0'; -fail_print: - ptr = strrchr(s, '\n'); - if (ptr) - *ptr = '\0'; - printf("%s%s%s\n", s, buf[0] ? " " : "", buf); -} - -/* get device list by ioctl */ -int jt_obd_list_ioctl(int argc, char **argv) -{ - int rc, index; - char rawbuf[MAX_IOC_BUFLEN], *buf = rawbuf; - struct obd_ioctl_data *data = (struct obd_ioctl_data *)buf; - - if (argc > 2) - return CMD_HELP; - /* Just ignore a -t option. Only supported with /proc. */ - else if (argc == 2 && strcmp(argv[1], "-t") != 0) - return CMD_HELP; - - for (index = 0;; index++) { - memset(buf, 0, sizeof(rawbuf)); - data->ioc_version = OBD_IOCTL_VERSION; - data->ioc_inllen1 = - sizeof(rawbuf) - __ALIGN_KERNEL(sizeof(*data), 8); - data->ioc_inlbuf1 = buf + __ALIGN_KERNEL(sizeof(*data), 8); - data->ioc_len = obd_ioctl_packlen(data); - data->ioc_count = index; - - rc = l_ioctl(OBD_DEV_ID, OBD_IOC_GETDEVICE, buf); - if (rc != 0) - break; - printf("%s\n", (char *)data->ioc_bulk); - } - if (rc != 0) { - if (errno == ENOENT) - /* no device or the last device */ - rc = 0; - else - fprintf(stderr, - "Error getting device list: %s: check dmesg\n", - strerror(errno)); - } - return rc; -} - -int jt_obd_list(int argc, char **argv) -{ - char buf[MAX_STRING_SIZE]; - int print_obd = 0; - glob_t path; - FILE *fp; - - if (argc > 2) - return CMD_HELP; - - if (argc == 2) { - if (strcmp(argv[1], "-t") == 0) - print_obd = 1; - else - return CMD_HELP; - } - - if (cfs_get_param_paths(&path, "devices") || - !(fp = fopen(path.gl_pathv[0], "r"))) { - cfs_free_param_data(&path); - - return jt_obd_list_ioctl(argc, argv); - } - - while (fgets(buf, sizeof(buf), fp) != NULL) - if (print_obd) - print_obd_line(buf); - else - printf("%s", buf); - - cfs_free_param_data(&path); - fclose(fp); - return 0; -} - struct jt_fid_space { __u64 jt_seq; __u64 jt_id; diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index 5c42bf49..d3f5908 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -134,7 +134,7 @@ int jt_llog_check(int argc, char **argv); struct lustre_cfg; int lcfg_ioctl(char * func, int dev_id, struct lustre_cfg *lcfg); int lcfg_mgs_ioctl(char *func, int dev_id, struct lustre_cfg *lcfg); -int parse_devname(char *func, char *name); +int parse_devname(char *func, char *name, int dev_id); char *jt_cmdname(char *func); -- 1.8.3.1