From: James Simmons Date: Mon, 29 Jan 2024 00:45:40 +0000 (-0500) Subject: LU-17479 utils: Update lnet tools to support PyYAML format X-Git-Tag: 2.15.61~51 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=9856b5e19dac4913f1b369bfce1a891a6263a0fb;p=fs%2Flustre-release.git LU-17479 utils: Update lnet tools to support PyYAML format The current cYAML implementation can't handle PyYAML indentation style. The reason is the underlying libyaml library creates different yaml events / tokens for the PyYAML format. I attempted to inject the missing yaml tokens from the PyYAML format but that failed to work. Also the tokens with the PyYAML produced the wrong scalar strings. I looked at moving to yaml events instead of tokens but that required a large change. The simplest change was to capture the YAML config input and place it into a locally allocated buffer. Then alter the location of '-' which changed the YAML config from PyYAML to something cYAML can handle. For this to work I needed to move the YAML config data handling out of cYAML_build_tree() to jt_import(). The reason was that lustre_yaml_cb_helper() is called more than once and for stdin it can only be read once. Test-Parameters: trivial testlist=sanity-lnet Change-Id: Ic8529ae264c9cbe6872da9a9e3421db78f8ea371 Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53845 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Chris Horn Reviewed-by: Serguei Smirnov Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index 10b3b4f..1d87f58 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -5798,7 +5798,8 @@ static cmd_handler_t lookup_fn(char *key, return NULL; } -static int lustre_yaml_cb_helper(char *f, struct lookup_cmd_hdlr_tbl *table, +static int lustre_yaml_cb_helper(char *f, int len, + struct lookup_cmd_hdlr_tbl *table, struct cYAML **show_rc, struct cYAML **err_rc) { struct cYAML *tree, *item = NULL, *head, *child; @@ -5806,7 +5807,7 @@ static int lustre_yaml_cb_helper(char *f, struct lookup_cmd_hdlr_tbl *table, char err_str[LNET_MAX_STR_LEN]; int rc = LUSTRE_CFG_RC_NO_ERR, return_rc = LUSTRE_CFG_RC_NO_ERR; - tree = cYAML_build_tree(f, NULL, 0, err_rc, false); + tree = cYAML_build_tree(NULL, f, len, err_rc, false); if (tree == NULL) return LUSTRE_CFG_RC_BAD_PARAM; @@ -5844,26 +5845,28 @@ out: return return_rc; } -int lustre_yaml_config(char *f, struct cYAML **err_rc) +int lustre_yaml_config(char *f, int len, struct cYAML **err_rc) { - return lustre_yaml_cb_helper(f, lookup_config_tbl, + return lustre_yaml_cb_helper(f, len, lookup_config_tbl, NULL, err_rc); } -int lustre_yaml_del(char *f, struct cYAML **err_rc) +int lustre_yaml_del(char *f, int len, struct cYAML **err_rc) { - return lustre_yaml_cb_helper(f, lookup_del_tbl, + return lustre_yaml_cb_helper(f, len, lookup_del_tbl, NULL, err_rc); } -int lustre_yaml_show(char *f, struct cYAML **show_rc, struct cYAML **err_rc) +int lustre_yaml_show(char *f, int len, struct cYAML **show_rc, + struct cYAML **err_rc) { - return lustre_yaml_cb_helper(f, lookup_show_tbl, + return lustre_yaml_cb_helper(f, len, lookup_show_tbl, show_rc, err_rc); } -int lustre_yaml_exec(char *f, struct cYAML **show_rc, struct cYAML **err_rc) +int lustre_yaml_exec(char *f, int len, struct cYAML **show_rc, + struct cYAML **err_rc) { - return lustre_yaml_cb_helper(f, lookup_exec_tbl, + return lustre_yaml_cb_helper(f, len, lookup_exec_tbl, show_rc, err_rc); } diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index a3708da..2fc7c08 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -746,7 +746,7 @@ int lustre_lnet_discover_nid(char *pnid, int force, int seq_no, * f - YAML file * err_rc - [OUT] struct cYAML tree describing the error. Freed by caller */ -int lustre_yaml_config(char *f, struct cYAML **err_rc); +int lustre_yaml_config(char *f, int len, struct cYAML **err_rc); /* * lustre_yaml_del @@ -756,7 +756,7 @@ int lustre_yaml_config(char *f, struct cYAML **err_rc); * f - YAML file * err_rc - [OUT] struct cYAML tree describing the error. Freed by caller */ -int lustre_yaml_del(char *f, struct cYAML **err_rc); +int lustre_yaml_del(char *f, int len, struct cYAML **err_rc); /* * lustre_yaml_show @@ -767,7 +767,7 @@ int lustre_yaml_del(char *f, struct cYAML **err_rc); * show_rc - [OUT] The show output in YAML. Must be freed by caller. * err_rc - [OUT] struct cYAML tree describing the error. Freed by caller */ -int lustre_yaml_show(char *f, struct cYAML **show_rc, +int lustre_yaml_show(char *f, int len, struct cYAML **show_rc, struct cYAML **err_rc); /* @@ -779,7 +779,7 @@ int lustre_yaml_show(char *f, struct cYAML **show_rc, * show_rc - [OUT] The show output in YAML. Must be freed by caller. * err_rc - [OUT] struct cYAML tree describing the error. Freed by caller */ -int lustre_yaml_exec(char *f, struct cYAML **show_rc, +int lustre_yaml_exec(char *f, int len, struct cYAML **show_rc, struct cYAML **err_rc); struct nid_node { diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index c37fcdf..ccc65e3 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -3681,8 +3682,12 @@ static int jt_import(int argc, char **argv) struct cYAML *err_rc = NULL; struct cYAML *show_rc = NULL; int rc = 0, return_rc = 0, opt, opt_found = 0; - char cmd = 'a'; - + char *yaml_blk = NULL, *buf, cmd = 'a'; + bool release = true; + char err_str[256]; + struct stat st; + FILE *input; + size_t len; const char *const short_options = "adseh"; static const struct option long_options[] = { { .name = "add", .has_arg = no_argument, .val = 'a' }, @@ -3690,7 +3695,8 @@ static int jt_import(int argc, char **argv) { .name = "show", .has_arg = no_argument, .val = 's' }, { .name = "exec", .has_arg = no_argument, .val = 'e' }, { .name = "help", .has_arg = no_argument, .val = 'h' }, - { .name = NULL } }; + { .name = NULL } + }; while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) != -1) { @@ -3728,28 +3734,97 @@ static int jt_import(int argc, char **argv) else if (!opt_found && argc == 2) file = argv[1]; + /* file always takes precedence */ + if (file != NULL) { + /* Set a file input. */ + input = fopen(file, "rb"); + if (!input) { + rc = -errno; + snprintf(err_str, sizeof(err_str), + "cannot open '%s': %s", file, + strerror(errno)); + cYAML_build_error(-1, -1, "yaml", "builder", + err_str, + &err_rc); + goto err; + } + } else { + input = stdin; + } + + /* assume that we're getting our input from stdin */ + rc = fstat(fileno(input), &st); + if (rc < 0) { + snprintf(err_str, sizeof(err_str), + "cannot get file stats '%s': %s", file, + strerror(-rc)); + cYAML_build_error(-1, -1, "yaml", "builder", + err_str, + &err_rc); + goto err; + } + + yaml_blk = buf = malloc(st.st_size); + if (!yaml_blk) { + rc = -ENOMEM; + snprintf(err_str, sizeof(err_str), + "failed to allocate buffer: %s", + strerror(-rc)); + cYAML_build_error(-1, -1, "yaml", "builder", + err_str, + &err_rc); + goto err; + } + len = st.st_size; + + while (fgets(buf, len, input) != NULL) { + char *seq = strstr(buf, "- "); + + if (seq) { + int skip; + + seq[0] = ' '; + skip = strspn(seq, " "); + if (skip) { + seq += skip - 2; + seq[0] = '-'; + } + /* PyYAML format has libyaml free the + * buffer for us. + */ + release = false; + } + buf += strlen(buf); + len -= strlen(buf); + } + switch (cmd) { case 'a': - rc = lustre_yaml_config(file, &err_rc); - return_rc = lustre_yaml_exec(file, &show_rc, &err_rc); + rc = lustre_yaml_config(yaml_blk, st.st_size, &err_rc); + return_rc = lustre_yaml_exec(yaml_blk, st.st_size, &show_rc, + &err_rc); cYAML_print_tree(show_rc); cYAML_free_tree(show_rc); break; case 'd': - rc = lustre_yaml_del(file, &err_rc); + rc = lustre_yaml_del(yaml_blk, st.st_size, &err_rc); break; case 's': - rc = lustre_yaml_show(file, &show_rc, &err_rc); + rc = lustre_yaml_show(yaml_blk, st.st_size, &show_rc, + &err_rc); cYAML_print_tree(show_rc); cYAML_free_tree(show_rc); break; case 'e': - rc = lustre_yaml_exec(file, &show_rc, &err_rc); + rc = lustre_yaml_exec(yaml_blk, st.st_size, &show_rc, + &err_rc); cYAML_print_tree(show_rc); cYAML_free_tree(show_rc); break; } - +err: + if (yaml_blk && release) + free(yaml_blk); if (rc || return_rc) { cYAML_print_tree2file(stderr, err_rc); cYAML_free_tree(err_rc);