LU-13366 utils: SEL yaml and copy file support

create a SEL file by a yaml template or by copying the template
from another file (setstripe --yaml and setstripe --copy).

Signed-off-by: Vitaly Fertman <c17818@cray.com>
Change-Id: I001226a9dcfa5706295cef1c13a2e7aeb5c498cc
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Alexander Boyko <c17825@cray.com>
Tested-by: Elena Gryaznova <c17455@cray.com>
Reviewed-on: https://es-gerrit.dev.cray.com/156286
HPE-bug-id: LUS-8039
Reviewed-on: https://review.whamcloud.com/37960
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Alexander Boyko <alexander.boyko@hpe.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh
index 923518f..08d2596 100644 (file)
--- a/lustre/tests/sanity-flr.sh
+++ b/lustre/tests/sanity-flr.sh
@@ -2061,6 +2061,7 @@ test_46() {
 
        ########################### 3. FLR file #############################
        echo "  ** 3. FLR file"
+       rm -f $file
        $LFS setstripe -N -E1m -S 1M -c2 -o0,1 -E4m -c1 -Eeof -N -E16m -Eeof \
                $file || error "3. Create FLR $file failed"
 
@@ -2089,6 +2090,38 @@ test_46() {
                error "6. setstripe FLR $dir failed"
 
        verify_46 $dir $dir.copy "6. FLR dir"
+
+       ########################### 7. SEL file ##############################
+       echo "  ** 7. SEL file"
+       rm -f $file
+       $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \
+               $file || error "Create $file failed"
+
+       rm -f $file.copy
+       verify_46 $file $file.copy "7. SEL file"
+
+       ########################### 8. SEL dir ##############################
+       echo "  ** 8. SEL dir"
+       $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \
+               $dir || error "setstripe $dir failed"
+
+       verify_46 $dir $dir.copy "8. SEL dir"
+
+       ########################### 9. FLR SEL file ##########################
+       echo "  ** 9. FLR SEL file"
+       rm -f $file
+       $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
+               -N -E1G -c4 -z128M -E-1 -z256M $file || error "Create $file failed"
+
+       rm -f $file.copy
+       verify_46 $file $file.copy "9. SEL file"
+
+       ########################### 10. FLR SEL dir #########################
+       echo "  ** 10. FLR SEL dir"
+       $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
+               -N -E1G -c4 -z128M -E-1 -z256M $dir || error "Create $file failed"
+
+       verify_46 $dir $dir.copy "10. SEL dir"
 }
 run_test 46 "Verify setstripe --copy option"
 

diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh
index 8e1882b..df4c1a0 100644 (file)
--- a/lustre/tests/sanity-pfl.sh
+++ b/lustre/tests/sanity-pfl.sh
@@ -851,6 +851,37 @@ test_16b() {
 }
 run_test 16b "Verify setstripe/getstripe with YAML config file + overstriping"
 
+test_16c() {
+       [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
+
+       local file=$DIR/$tdir/$tfile
+       local dir=$DIR/$tdir/dir
+       local temp=$DIR/$tdir/template
+       rm -rf $DIR/$tdir
+       test_mkdir $DIR/$tdir
+
+       #####################################################################
+       #                           1. SEL file
+       # set stripe for source file
+       $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \
+               $file || error "Create $file failed"
+
+       echo "1. SEL file"
+       verify_yaml_layout $file $file.copy $temp "1. PFL file"
+
+       #####################################################################
+       #                           2. SEL dir
+       # set stripe for source dir
+       test_mkdir $dir
+       $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \
+               $dir || error "setstripe $dir failed"
+
+       test_mkdir $dir.copy
+       echo "2. SEL template on dir"
+       verify_yaml_layout $dir $dir.copy $temp.dir "2. PFL dir"
+}
+run_test 16c "Verify setstripe/getstripe for SEL layout with YAML config file"
+
 test_17() {
        [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
        local file=$DIR/$tdir/$tfile

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 4f6edf1..8b512ca 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -2625,26 +2625,75 @@ static int build_component(struct llapi_layout **layout,
        return rc;
 }
 
+static int build_prev_component(struct llapi_layout **layout,
+                               struct lfs_setstripe_args *prev,
+                               struct lfs_setstripe_args *lsa,
+                               bool set_extent)
+{
+       int extension = lsa->lsa_comp_flags & LCME_FL_EXTENSION;
+       int rc;
+
+       if (prev->lsa_stripe_size) {
+               if (extension) {
+                       prev->lsa_comp_end = lsa->lsa_comp_end;
+                       prev->lsa_extension_size = lsa->lsa_extension_size;
+                       prev->lsa_extension_comp = true;
+               }
+
+               rc = build_component(layout, prev, true);
+               if (rc)
+                       return rc;
+       }
+
+       /* Copy lsa to previous lsa;
+        * if this is an extension component, make the previous invalid; */
+       if (extension)
+               prev->lsa_stripe_size = 0;
+       else
+               *prev = *lsa;
+
+       return 0;
+}
+
 static int build_layout_from_yaml_node(struct cYAML *node,
                                       struct llapi_layout **layout,
                                       struct lfs_setstripe_args *lsa,
-                                      __u32 *osts)
+                                      struct lfs_setstripe_args *prevp)
 {
+       struct lfs_setstripe_args prev = { 0 };
+       __u32 *osts = lsa->lsa_tgts;
        char *string;
        int rc = 0;
 
+       if (prevp == NULL)
+               prevp = &prev;
+
        while (node) {
+               string = node->cy_string;
+
                if (node->cy_type == CYAML_TYPE_OBJECT) {
                        /* go deep to sub blocks */
+                       if (string && !strncmp(string, "component", 9) &&
+                           strncmp(string, "component0", 10) &&
+                           strncmp(string, "components", 10)) {
+                               rc = build_prev_component(layout, prevp, lsa, true);
+                               if (rc)
+                                       return rc;
+
+                               /* initialize lsa. */
+                               setstripe_args_init(lsa);
+                               lsa->lsa_first_comp = false;
+                               lsa->lsa_tgts = osts;
+                       }
+
                        rc = build_layout_from_yaml_node(node->cy_child, layout,
-                                                        lsa, osts);
+                                                        lsa, prevp);
                        if (rc)
                                return rc;
                } else {
                        if (node->cy_string == NULL)
                                return -EINVAL;
 
-                       string = node->cy_string;
                        /* skip leading lmm_ if present, to simplify parsing */
                        if (strncmp(string, "lmm_", 4) == 0)
                                string += 4;
@@ -2678,18 +2727,8 @@ static int build_layout_from_yaml_node(struct cYAML *node,
                                if (!strcmp(string, "lcm_mirror_count")) {
                                        lsa->lsa_mirror_count = node->cy_valueint;
                                } else if (!strcmp(string, "lcme_extent.e_start")) {
-                                       if (node->cy_valueint != 0 || *layout != NULL) {
-                                               rc = build_component(layout, lsa, true);
-                                               if (rc)
-                                                       return rc;
-                                       }
-
                                        if (node->cy_valueint == 0)
                                                lsa->lsa_first_comp = true;
-
-                                       /* initialize lsa */
-                                       setstripe_args_init(lsa);
-                                       lsa->lsa_tgts = osts;
                                } else if (!strcmp(string, "lcme_extent.e_end")) {
                                        if (node->cy_valueint == -1)
                                                lsa->lsa_comp_end = LUSTRE_EOF;
@@ -2699,6 +2738,9 @@ static int build_layout_from_yaml_node(struct cYAML *node,
                                        lsa->lsa_stripe_count = node->cy_valueint;
                                } else if (!strcmp(string, "stripe_size")) {
                                        lsa->lsa_stripe_size = node->cy_valueint;
+                               } else if (!strcmp(string, "extension_size")) {
+                                       lsa->lsa_extension_size = node->cy_valueint;
+                                       lsa->lsa_extension_comp = true;
                                } else if (!strcmp(string, "stripe_offset")) {
                                        lsa->lsa_stripe_off = node->cy_valueint;
                                } else if (!strcmp(string, "l_ost_idx")) {
@@ -2710,6 +2752,15 @@ static int build_layout_from_yaml_node(struct cYAML *node,
                node = node->cy_next;
        }
 
+       if (prevp == &prev) {
+               rc = build_prev_component(layout, prevp, lsa, true);
+               if (rc)
+                       return rc;
+
+               if (!(lsa->lsa_comp_flags & LCME_FL_EXTENSION))
+                       rc = build_component(layout, lsa, *layout != NULL);
+       }
+
        return rc;
 }
 
@@ -2737,13 +2788,11 @@ static int lfs_comp_create_from_yaml(char *template,
        setstripe_args_init(lsa);
        lsa->lsa_tgts = osts;
 
-       rc = build_layout_from_yaml_node(tree, layout, lsa, osts);
+       rc = build_layout_from_yaml_node(tree, layout, lsa, NULL);
        if (rc) {
                fprintf(stderr, "%s: cannot build layout from YAML file %s.\n",
                        progname, template);
                goto err;
-       } else {
-               rc = build_component(layout, lsa, *layout != NULL);
        }
        /* clean clean lsa */
        setstripe_args_init(lsa);
@@ -2842,7 +2891,7 @@ static int layout_adjust_first_extent(char *fname, struct llapi_layout *layout,
        uint64_t end;
        int rc, ret = 0;
 
-       if (layout == NULL)
+       if (layout == NULL || !(comp_add || llapi_layout_is_composite(layout)))
                return 0;
 
        errno = 0;
@@ -3758,21 +3807,6 @@ static int lfs_setstripe_internal(int argc, char **argv,
                }
        }
 
-       if (layout != NULL || mirror_list != NULL) {
-               if (mirror_list)
-                       result = mirror_adjust_first_extents(mirror_list);
-               else
-                       result = layout_adjust_first_extent(fname, layout,
-                                                           comp_add);
-               if (result == -ENODATA)
-                       comp_add = 0;
-               else if (result != 0) {
-                       fprintf(stderr, "error: %s: invalid layout\n",
-                               progname);
-                       goto error;
-               }
-       }
-
        if (from_yaml && from_copy) {
                fprintf(stderr,
                        "%s: can't specify --yaml and --copy together\n",
@@ -3925,6 +3959,21 @@ static int lfs_setstripe_internal(int argc, char **argv,
                }
        }
 
+       if (layout != NULL || mirror_list != NULL) {
+               if (mirror_list)
+                       result = mirror_adjust_first_extents(mirror_list);
+               else
+                       result = layout_adjust_first_extent(fname, layout,
+                                                           comp_add);
+               if (result == -ENODATA)
+                       comp_add = 0;
+               else if (result != 0) {
+                       fprintf(stderr, "error: %s: invalid layout\n",
+                               progname);
+                       goto error;
+               }
+       }
+
        for (fname = argv[optind]; fname != NULL; fname = argv[++optind]) {
                if (from_copy) {
                        layout = llapi_layout_get_by_path(template ?: fname, 0);