From: Vitaly Fertman Date: Fri, 13 Mar 2020 13:03:08 +0000 (+0300) Subject: LU-13366 utils: SEL yaml and copy file support X-Git-Tag: 2.13.54~49 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=ee916af10de2f9e1dd8b79d1f08a78401277ef96;ds=sidebyside LU-13366 utils: SEL yaml and copy file support create a SEL file by a yaml template or by copying the template from another file (setstripe --yaml and setstripe --copy). Signed-off-by: Vitaly Fertman Change-Id: I001226a9dcfa5706295cef1c13a2e7aeb5c498cc Reviewed-by: Alexey Lyashkov Reviewed-by: Alexander Boyko Tested-by: Elena Gryaznova Reviewed-on: https://es-gerrit.dev.cray.com/156286 HPE-bug-id: LUS-8039 Reviewed-on: https://review.whamcloud.com/37960 Reviewed-by: Andreas Dilger Reviewed-by: Alexander Boyko Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 923518f..08d2596 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -2061,6 +2061,7 @@ test_46() { ########################### 3. FLR file ############################# echo " ** 3. FLR file" + rm -f $file $LFS setstripe -N -E1m -S 1M -c2 -o0,1 -E4m -c1 -Eeof -N -E16m -Eeof \ $file || error "3. Create FLR $file failed" @@ -2089,6 +2090,38 @@ test_46() { error "6. setstripe FLR $dir failed" verify_46 $dir $dir.copy "6. FLR dir" + + ########################### 7. SEL file ############################## + echo " ** 7. SEL file" + rm -f $file + $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \ + $file || error "Create $file failed" + + rm -f $file.copy + verify_46 $file $file.copy "7. SEL file" + + ########################### 8. SEL dir ############################## + echo " ** 8. SEL dir" + $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \ + $dir || error "setstripe $dir failed" + + verify_46 $dir $dir.copy "8. SEL dir" + + ########################### 9. FLR SEL file ########################## + echo " ** 9. FLR SEL file" + rm -f $file + $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \ + -N -E1G -c4 -z128M -E-1 -z256M $file || error "Create $file failed" + + rm -f $file.copy + verify_46 $file $file.copy "9. SEL file" + + ########################### 10. FLR SEL dir ######################### + echo " ** 10. FLR SEL dir" + $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \ + -N -E1G -c4 -z128M -E-1 -z256M $dir || error "Create $file failed" + + verify_46 $dir $dir.copy "10. SEL dir" } run_test 46 "Verify setstripe --copy option" diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 8e1882b..df4c1a0 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -851,6 +851,37 @@ test_16b() { } run_test 16b "Verify setstripe/getstripe with YAML config file + overstriping" +test_16c() { + [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" + + local file=$DIR/$tdir/$tfile + local dir=$DIR/$tdir/dir + local temp=$DIR/$tdir/template + rm -rf $DIR/$tdir + test_mkdir $DIR/$tdir + + ##################################################################### + # 1. SEL file + # set stripe for source file + $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \ + $file || error "Create $file failed" + + echo "1. SEL file" + verify_yaml_layout $file $file.copy $temp "1. PFL file" + + ##################################################################### + # 2. SEL dir + # set stripe for source dir + test_mkdir $dir + $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \ + $dir || error "setstripe $dir failed" + + test_mkdir $dir.copy + echo "2. SEL template on dir" + verify_yaml_layout $dir $dir.copy $temp.dir "2. PFL dir" +} +run_test 16c "Verify setstripe/getstripe for SEL layout with YAML config file" + test_17() { [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" local file=$DIR/$tdir/$tfile diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 4f6edf1..8b512ca 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -2625,26 +2625,75 @@ static int build_component(struct llapi_layout **layout, return rc; } +static int build_prev_component(struct llapi_layout **layout, + struct lfs_setstripe_args *prev, + struct lfs_setstripe_args *lsa, + bool set_extent) +{ + int extension = lsa->lsa_comp_flags & LCME_FL_EXTENSION; + int rc; + + if (prev->lsa_stripe_size) { + if (extension) { + prev->lsa_comp_end = lsa->lsa_comp_end; + prev->lsa_extension_size = lsa->lsa_extension_size; + prev->lsa_extension_comp = true; + } + + rc = build_component(layout, prev, true); + if (rc) + return rc; + } + + /* Copy lsa to previous lsa; + * if this is an extension component, make the previous invalid; */ + if (extension) + prev->lsa_stripe_size = 0; + else + *prev = *lsa; + + return 0; +} + static int build_layout_from_yaml_node(struct cYAML *node, struct llapi_layout **layout, struct lfs_setstripe_args *lsa, - __u32 *osts) + struct lfs_setstripe_args *prevp) { + struct lfs_setstripe_args prev = { 0 }; + __u32 *osts = lsa->lsa_tgts; char *string; int rc = 0; + if (prevp == NULL) + prevp = &prev; + while (node) { + string = node->cy_string; + if (node->cy_type == CYAML_TYPE_OBJECT) { /* go deep to sub blocks */ + if (string && !strncmp(string, "component", 9) && + strncmp(string, "component0", 10) && + strncmp(string, "components", 10)) { + rc = build_prev_component(layout, prevp, lsa, true); + if (rc) + return rc; + + /* initialize lsa. */ + setstripe_args_init(lsa); + lsa->lsa_first_comp = false; + lsa->lsa_tgts = osts; + } + rc = build_layout_from_yaml_node(node->cy_child, layout, - lsa, osts); + lsa, prevp); if (rc) return rc; } else { if (node->cy_string == NULL) return -EINVAL; - string = node->cy_string; /* skip leading lmm_ if present, to simplify parsing */ if (strncmp(string, "lmm_", 4) == 0) string += 4; @@ -2678,18 +2727,8 @@ static int build_layout_from_yaml_node(struct cYAML *node, if (!strcmp(string, "lcm_mirror_count")) { lsa->lsa_mirror_count = node->cy_valueint; } else if (!strcmp(string, "lcme_extent.e_start")) { - if (node->cy_valueint != 0 || *layout != NULL) { - rc = build_component(layout, lsa, true); - if (rc) - return rc; - } - if (node->cy_valueint == 0) lsa->lsa_first_comp = true; - - /* initialize lsa */ - setstripe_args_init(lsa); - lsa->lsa_tgts = osts; } else if (!strcmp(string, "lcme_extent.e_end")) { if (node->cy_valueint == -1) lsa->lsa_comp_end = LUSTRE_EOF; @@ -2699,6 +2738,9 @@ static int build_layout_from_yaml_node(struct cYAML *node, lsa->lsa_stripe_count = node->cy_valueint; } else if (!strcmp(string, "stripe_size")) { lsa->lsa_stripe_size = node->cy_valueint; + } else if (!strcmp(string, "extension_size")) { + lsa->lsa_extension_size = node->cy_valueint; + lsa->lsa_extension_comp = true; } else if (!strcmp(string, "stripe_offset")) { lsa->lsa_stripe_off = node->cy_valueint; } else if (!strcmp(string, "l_ost_idx")) { @@ -2710,6 +2752,15 @@ static int build_layout_from_yaml_node(struct cYAML *node, node = node->cy_next; } + if (prevp == &prev) { + rc = build_prev_component(layout, prevp, lsa, true); + if (rc) + return rc; + + if (!(lsa->lsa_comp_flags & LCME_FL_EXTENSION)) + rc = build_component(layout, lsa, *layout != NULL); + } + return rc; } @@ -2737,13 +2788,11 @@ static int lfs_comp_create_from_yaml(char *template, setstripe_args_init(lsa); lsa->lsa_tgts = osts; - rc = build_layout_from_yaml_node(tree, layout, lsa, osts); + rc = build_layout_from_yaml_node(tree, layout, lsa, NULL); if (rc) { fprintf(stderr, "%s: cannot build layout from YAML file %s.\n", progname, template); goto err; - } else { - rc = build_component(layout, lsa, *layout != NULL); } /* clean clean lsa */ setstripe_args_init(lsa); @@ -2842,7 +2891,7 @@ static int layout_adjust_first_extent(char *fname, struct llapi_layout *layout, uint64_t end; int rc, ret = 0; - if (layout == NULL) + if (layout == NULL || !(comp_add || llapi_layout_is_composite(layout))) return 0; errno = 0; @@ -3758,21 +3807,6 @@ static int lfs_setstripe_internal(int argc, char **argv, } } - if (layout != NULL || mirror_list != NULL) { - if (mirror_list) - result = mirror_adjust_first_extents(mirror_list); - else - result = layout_adjust_first_extent(fname, layout, - comp_add); - if (result == -ENODATA) - comp_add = 0; - else if (result != 0) { - fprintf(stderr, "error: %s: invalid layout\n", - progname); - goto error; - } - } - if (from_yaml && from_copy) { fprintf(stderr, "%s: can't specify --yaml and --copy together\n", @@ -3925,6 +3959,21 @@ static int lfs_setstripe_internal(int argc, char **argv, } } + if (layout != NULL || mirror_list != NULL) { + if (mirror_list) + result = mirror_adjust_first_extents(mirror_list); + else + result = layout_adjust_first_extent(fname, layout, + comp_add); + if (result == -ENODATA) + comp_add = 0; + else if (result != 0) { + fprintf(stderr, "error: %s: invalid layout\n", + progname); + goto error; + } + } + for (fname = argv[optind]; fname != NULL; fname = argv[++optind]) { if (from_copy) { layout = llapi_layout_get_by_path(template ?: fname, 0);