LU-15565 utils: updated lfs getstripe yaml format

For composite files 'lfs getstripe --yaml' used a key for each
component rather than an array. This hindered the use of the
output with common YAML tooling. The leading spaces meant users
needed to pre-process the output for seemingly no reason. Each
component also had the id harcoded into the name rather than
using a more flexible list ('components:').

'lfs setstripe --yaml YAML_FILE file|dir' is still compatible
with the previous YAML format incase there were stored files
being used.

The new YAML formatting has been check with results from
"https://zhwt.github.io/yaml-to-go/".

Modified the argument parsing in 'lfs_migrate' to be more
robust as it was failing tests 56[w[a-c],x[b,c]].

Signed-off-by: Frederick Dilger <fdilger@whamcloud.com>
Change-Id: I50ff4ebd9413fb66f05647c11542f7ce9f1ba879
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55311
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index 16e61f5..c81175f 100755 (executable)
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -379,21 +379,33 @@ lfs_migrate() {
                        layout+="--copy $olddir"
                        OPT_COPY=true
                elif ! $OPT_COMP; then
+                       # this formatting is no longer consistent LU-15565
                        # avoid multiple getstripe calls
                        #   lcm_mirror_count:  1
                        #   lcm_entry_count:   0
                        #      lmm_stripe_count:  1
                        #      lmm_stripe_size:   1048576
                        #      lmm_pool:          pool_abc
-                       local l_mirror_count=0
-                       local l_comp_count=1
-                       local l_stripe_count=2
-                       local l_stripe_size=3
-                       local l_stripe_pool=4
+                       local l_mirror_count
+                       local l_comp_count
+                       local l_stripe_count
+                       local l_stripe_size
+                       local l_stripe_pool
                        local layout_info
 
                        layout_info=($($LFS getstripe $getstripe_opts $OLDNAME \
-                                      2>/dev/null | awk '{ print $2 }'))
+                                      2>/dev/null | awk '{ print }'))
+
+                       for ((i = 0; i < ${#layout_info[*]}; i++)); do
+                               case "${layout_info[$i]}" in
+                               *mirror_count*) l_mirror_count=$((i + 1));;
+                               *entry_count*) l_comp_count=$((i + 1));;
+                               *stripe_count*) l_stripe_count=$((i + 1));;
+                               *stripe_size*) l_stripe_size=$((i + 1));;
+                               *stripe_pool*) l_stripe_pool=$((i + 1));;
+                               esac
+                       done
+
                        # If rsync copies Lustre xattrs properly in the future
                        # (i.e. before the file data, so that it preserves
                        # striping) then we don't need this getstripe stuff.

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 36c3c7d..8cc95af 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -21316,6 +21316,135 @@ test_205i() {
 }
 run_test 205i "check job_xattr parameter accepts and rejects values correctly"
 
+# LU-15565
+test_205j() {
+       local new=$DIR/$tdir/$tfile-layout-new
+       local old=$DIR/$tdir/$tfile-layout-old
+
+       test_mkdir $DIR/$tdir
+
+       cat << PFL_LAYOUT_NEW > $new
+lcm_layout_gen:    3
+lcm_mirror_count:  1
+lcm_entry_count:   3
+components:
+  - lcme_id:             1
+    lcme_mirror_id:      0
+    lcme_flags:          init
+    lcme_extent.e_start: 0
+    lcme_extent.e_end:   1073741824
+    sub_layout:
+      lmm_stripe_count:  1
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: 0
+      lmm_objects:
+      - l_ost_idx: 0
+        l_fid:     0x280000400:0x2:0x0
+  - lcme_id:             2
+    lcme_mirror_id:      0
+    lcme_flags:          0
+    lcme_extent.e_start: 1073741824
+    lcme_extent.e_end:   4294967296
+    sub_layout:
+      lmm_stripe_count:  4
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: -1
+  - lcme_id:             3
+    lcme_mirror_id:      0
+    lcme_flags:          0
+    lcme_extent.e_start: 4294967296
+    lcme_extent.e_end:   EOF
+    sub_layout:
+      lmm_stripe_count:  4
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: -1
+PFL_LAYOUT_NEW
+
+# Layout changed in 2.16.0
+       cat << PFL_LAYOUT_OLD > $old
+  lcm_layout_gen:    3
+  lcm_mirror_count:  1
+  lcm_entry_count:   3
+  component0:
+    lcme_id:             1
+    lcme_mirror_id:      0
+    lcme_flags:          init
+    lcme_extent.e_start: 0
+    lcme_extent.e_end:   1073741824
+    sub_layout:
+      lmm_stripe_count:  1
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: 0
+      lmm_objects:
+      - l_ost_idx: 0
+        l_fid:     0x280000400:0x2:0x0
+  component1:
+    lcme_id:             2
+    lcme_mirror_id:      0
+    lcme_flags:          0
+    lcme_extent.e_start: 1073741824
+    lcme_extent.e_end:   4294967296
+    sub_layout:
+      lmm_stripe_count:  4
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: -1
+  component2:
+    lcme_id:             3
+    lcme_mirror_id:      0
+    lcme_flags:          0
+    lcme_extent.e_start: 4294967296
+    lcme_extent.e_end:   EOF
+    sub_layout:
+      lmm_stripe_count:  4
+      lmm_stripe_size:   4194304
+      lmm_pattern:       raid0
+      lmm_layout_gen:    0
+      lmm_stripe_offset: -1
+PFL_LAYOUT_OLD
+
+       local yaml_file=$DIR/$tdir/layout.yaml
+       local dir=$DIR/$tdir
+       local test=$dir/$tfile.test
+
+       $LFS setstripe -E 4M -c 2 -E EOF -c 4 $test ||
+               error "failed to setstripe"
+       $LFS getstripe --yaml $test > $yaml_file ||
+               error "failed to getstripe --yaml"
+
+       $LFS setstripe --yaml $old $dir/$tfile.new ||
+               error "failed to setstripe from new YAML format"
+       # test that the old format can still be used
+       $LFS setstripe --yaml $old $dir/$tfile.old ||
+               error "failed to setstripe from old YAML format"
+
+       local orig=$(get_layout_param $test)
+       local rest=$(cat $yaml_file | parse_layout_param)
+       [[ "$orig" == "$rest" ]] ||
+               error "failed to parse current YAML layout"
+
+       orig=$(get_layout_param $dir/$tfile.old)
+       rest=$(cat $old | parse_layout_param)
+       [[ "$orig" == "$rest" ]] ||
+               error "failed to parse old YAML layout"
+
+       orig=$(get_layout_param $dir/$tfile.new)
+       rest=$(cat $new | parse_layout_param)
+       [[ "$orig" == "$rest" ]] ||
+               error "failed to parse new YAML layout"
+
+}
+run_test 205j "verify new YAML format is valid and back-compatible"
+
 # LU-1480, LU-1773 and LU-1657
 test_206() {
        mkdir -p $DIR/$tdir

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 5cb6c33..5eb9628 100755 (executable)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -3047,8 +3047,7 @@ static int build_component(struct llapi_layout **layout,
 
 static int build_prev_component(struct llapi_layout **layout,
                                struct lfs_setstripe_args *prev,
-                               struct lfs_setstripe_args *lsa,
-                               bool set_extent)
+                               struct lfs_setstripe_args *lsa)
 {
        int extension = lsa->lsa_comp_flags & LCME_FL_EXTENSION;
        int rc;
@@ -3098,22 +3097,9 @@ static int build_layout_from_yaml_node(struct cYAML *node,
        while (node) {
                string = node->cy_string;
 
-               if (node->cy_type == CYAML_TYPE_OBJECT) {
+               if (node->cy_type == CYAML_TYPE_OBJECT ||
+                   node->cy_type == CYAML_TYPE_ARRAY) {
                        /* go deep to sub blocks */
-                       if (string && !strncmp(string, "component", 9) &&
-                           strncmp(string, "component0", 10) &&
-                           strncmp(string, "components", 10)) {
-                               rc = build_prev_component(layout, prevp, lsa,
-                                                         true);
-                               if (rc)
-                                       return rc;
-
-                               /* initialize lsa. */
-                               setstripe_args_init(lsa);
-                               lsa->lsa_first_comp = false;
-                               lsa->lsa_tgts = osts;
-                       }
-
                        rc = build_layout_from_yaml_node(node->cy_child, layout,
                                                         lsa, prevp);
                        if (rc)
@@ -3122,6 +3108,17 @@ static int build_layout_from_yaml_node(struct cYAML *node,
                        if (!node->cy_string)
                                return -EINVAL;
 
+                       if (strcmp(string, "lcme_id") == 0 &&
+                           lsa->lsa_stripe_count != LLAPI_LAYOUT_DEFAULT) {
+                               rc = build_prev_component(layout, prevp, lsa);
+                               if (rc)
+                                       return rc;
+
+                               /* initialize lsa. */
+                               setstripe_args_init(lsa);
+                               lsa->lsa_first_comp = false;
+                               lsa->lsa_tgts = osts;
+                       }
                        /* skip leading lmm_ if present, to simplify parsing */
                        if (strncmp(string, "lmm_", 4) == 0)
                                string += 4;
@@ -3182,7 +3179,7 @@ static int build_layout_from_yaml_node(struct cYAML *node,
        }
 
        if (prevp == &prev) {
-               rc = build_prev_component(layout, prevp, lsa, true);
+               rc = build_prev_component(layout, prevp, lsa);
                if (rc)
                        return rc;
 

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 009b703..7cb4d63 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -2964,7 +2964,8 @@ static void lov_dump_user_lmm_v1v3(struct lov_user_md *lum, char *pool_name,
                        }
                }
        }
-       llapi_printf(LLAPI_MSG_NORMAL, "\n");
+       if (!yaml)
+               llapi_printf(LLAPI_MSG_NORMAL, "\n");
 }
 
 static void hsm_flags2str(__u32 hsm_flags)
@@ -3254,15 +3255,15 @@ static void lov_dump_comp_v1_header(struct find_param *param, char *path,
 
        if (verbose & VERBOSE_GENERATION) {
                if (verbose & ~VERBOSE_GENERATION)
-                       llapi_printf(LLAPI_MSG_NORMAL, "%2slcm_layout_gen:    ",
-                                    " ");
+                       llapi_printf(LLAPI_MSG_NORMAL, "%slcm_layout_gen:    ",
+                                    yaml ? "" : "  ");
                llapi_printf(LLAPI_MSG_NORMAL, "%u\n", comp_v1->lcm_layout_gen);
        }
 
        if (verbose & VERBOSE_MIRROR_COUNT) {
                if (verbose & ~VERBOSE_MIRROR_COUNT)
-                       llapi_printf(LLAPI_MSG_NORMAL, "%2slcm_mirror_count:  ",
-                                    " ");
+                       llapi_printf(LLAPI_MSG_NORMAL, "%slcm_mirror_count:  ",
+                                    yaml ? "" : "  ");
                llapi_printf(LLAPI_MSG_NORMAL, "%u\n",
                             comp_v1->lcm_magic == LOV_USER_MAGIC_COMP_V1 ?
                             comp_v1->lcm_mirror_count + 1 : 1);
@@ -3270,14 +3271,14 @@ static void lov_dump_comp_v1_header(struct find_param *param, char *path,
 
        if (verbose & VERBOSE_COMP_COUNT) {
                if (verbose & ~VERBOSE_COMP_COUNT)
-                       llapi_printf(LLAPI_MSG_NORMAL, "%2slcm_entry_count:   ",
-                                    " ");
+                       llapi_printf(LLAPI_MSG_NORMAL, "%slcm_entry_count:   ",
+                                    yaml ? "" : "  ");
                llapi_printf(LLAPI_MSG_NORMAL, "%u\n",
                             comp_v1->lcm_magic == LOV_USER_MAGIC_COMP_V1 ?
                             comp_v1->lcm_entry_count : 0);
        }
 
-       if (verbose & VERBOSE_DETAIL && !yaml)
+       if (verbose & VERBOSE_DETAIL || yaml)
                llapi_printf(LLAPI_MSG_NORMAL, "components:\n");
 }
 
@@ -3320,11 +3321,8 @@ static void lov_dump_comp_v1_entry(struct find_param *param,
 
        entry = &comp_v1->lcm_entries[index];
 
-       if (yaml)
-               llapi_printf(LLAPI_MSG_NORMAL, "%2scomponent%d:\n", " ", index);
-
-       if (verbose & VERBOSE_COMP_ID) {
-               if (verbose & VERBOSE_DETAIL && !yaml)
+       if (verbose & VERBOSE_COMP_ID || yaml) {
+               if (verbose & VERBOSE_DETAIL || yaml)
                        llapi_printf(LLAPI_MSG_NORMAL,
                                     "%slcme_id:             ", "  - ");
                else if (verbose & ~VERBOSE_COMP_ID)