LU-10070 utils: SEL: lfs find & getstripe support

The support includes:
- add --extension-size option to lfs find & getstripe along
  with +/- functionality;
- do not take the extension components into account for
  lfs find --stripe-size and --stripe-count;
- add appropriate tests;

Cray-bug-id: LUS-2528
Signed-off-by: Vitaly Fertman <c17818@cray.com>
Change-Id: Ic3ad4c713e8c676998cf7d02b524ba266c992924
Reviewed-on: https://review.whamcloud.com/34909
Reviewed-by: Patrick Farrell <pfarrell@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1
index f44a366..92f879b 100644 (file)
--- a/lustre/doc/lfs-find.1
+++ b/lustre/doc/lfs-find.1
@@ -10,6 +10,7 @@ lfs-find \- Lustre client utility to list files with specific attributes
       [[\fB!\fR] \fB--component-end|\fB--comp-end\fR|\fB-E\fR [\fB+-\fR]\fIn\fR[\fBKMGTPE\fR]]
       [[\fB!\fR] \fB--component-flags|\fB--comp-flags\fR <[^]\fIflag\fB,\fR...>]
       [[\fB!\fR] \fB--component-start|\fB--comp-start\fR [\fB+-\fR]\fIn\fR[\fBKMGTPE\fR]]
+      [[\fB!\fR] \fB--extension-size|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]]
       [[\fB!\fR] \fB--gid\fR|\fB-g\fR|\fB--group\fR|\fB-G\fR <\fIgname\fR>|<\fIgid\fR>]
       [[\fB!\fR] \fB--layout\fR|\fB-L mdt\fR,\fBraid0\fR,\fBreleased\fR]
       [[\fB!\fR] \fB--foreign\fR [<\fItype\fR>]]
@@ -244,7 +245,13 @@ are multiple OSTs that are being replaced.
 Stripe size is \fIn\fR bytes, or \fBK\fRibi-, \fBM\fRebi-,
 \fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-abytes if a
 suffix is given.  For composite files, this matches the stripe
-size of the last initialized component.
+size of any non-extension component.
+.TP
+.BR --extension-size | --ext-size | -z
+Extension size is \fIn\fR bytes, or \fBK\fRibi-, \fBM\fRebi-,
+\fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-abytes if a
+suffix is given.  For composite files, this matches the extension
+size of any extension component.
 .TP
 .BR --type | -t
 File has type: \fBb\fRlock, \fBc\fRharacter, \fBd\fRirectory,

diff --git a/lustre/doc/lfs-getstripe.1 b/lustre/doc/lfs-getstripe.1
index 7c4190b..9321c69 100644 (file)
--- a/lustre/doc/lfs-getstripe.1
+++ b/lustre/doc/lfs-getstripe.1
@@ -8,6 +8,7 @@ lfs getstripe \- Lustre client command to print layout parameters of a file
       [\fB--component-flags\fR|\fB--comp-flags\fR[\fB=\fIflags\fR]]
       [\fB--component-id\fR|\fB--comp-id\fR[=\fIid\fR]|\fB-I\fR[\fIid\fR]]
       [\fB--component-start\fR[\fB=\fR[\fB+-\fR]\fIstart\fR[\fBKMGTPE\fR]]]
+      [\fB--extension-size\fR|\fB--ext-size\fR|\fB-z\fR]
       [\fB--directory\fR|\fB-d\fR]
 [\fB--fid\fR|\fB-F\fR]
 [\fB--generation\fR|\fB-g\fR]
@@ -21,10 +22,10 @@ lfs getstripe \- Lustre client command to print layout parameters of a file
 [\fB--stripe-count\fR|\fB-c\fR]
 [\fB--stripe-index\fR|\fB-i\fR]
       [\fB--stripe-size\fR|\fB-S\fR]
-[[\fB!\fR] \fB--mirror-index\fR=[\fB+-\fR]\fI<index>\fR | [\fB!\fR] \fB--mirror-id\fR=[\fB+-\fR]\fI<id>\fR]
-[\fB--verbose\fR|\fB-v\fR]
+      [[\fB!\fR] \fB--mirror-index\fR=[\fB+-\fR]\fI<index>\fR | [\fB!\fR] \fB--mirror-id\fR=[\fB+-\fR]\fI<id>\fR]
+      [\fB--verbose\fR|\fB-v\fR]
 [\fB--yaml\fR|\fB-y\fR]
-      <\fIdirname\fR|\fIfilename\fR> ...
+<\fIdirname\fR|\fIfilename\fR> ...
 .SH DESCRIPTION
 .B lfs getstripe
 is used to list the layout/striping information for a given filename or
@@ -33,6 +34,7 @@ and allocated OST objects for each file will be shown. If you only want
 specific layout information to be printed, then the
 .BR --stripe-count ,
 .BR --stripe-size ,
+.BR --extension-size ,
 .BR --stripe-index ,
 .BR --layout ,
 .BR --fid ,
@@ -235,6 +237,10 @@ Print the starting OST index for the file layout.
 Print the stripe size in bytes.  For composite files this is the stripe
 size of the last initialized component.
 .TP
+.BR --extension-size | --ext-size | -z
+Print the extension size in bytes. For composite files this is the extension
+size of the first extension component.
+.TP
 .BR --verbose | -v
 Also print the layout magic, FID sequence, FID object ID, and FID, in
 addition to the normally-printed attributes.

diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index 6ae3e66..97d10bb 100644 (file)
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -172,6 +172,7 @@ enum llapi_layout_verbose  {
        VERBOSE_HASH_TYPE       =  0x8000,
        VERBOSE_MIRROR_COUNT    = 0x10000,
        VERBOSE_MIRROR_ID       = 0x20000,
+       VERBOSE_EXT_SIZE        = 0x40000,
        VERBOSE_DEFAULT         = VERBOSE_STRIPE_COUNT | VERBOSE_STRIPE_SIZE |
                                  VERBOSE_STRIPE_OFFSET | VERBOSE_POOL |
                                  VERBOSE_OBJID | VERBOSE_GENERATION |
@@ -179,7 +180,7 @@ enum llapi_layout_verbose  {
                                  VERBOSE_COMP_COUNT | VERBOSE_COMP_FLAGS |
                                  VERBOSE_COMP_START | VERBOSE_COMP_END |
                                  VERBOSE_COMP_ID | VERBOSE_MIRROR_COUNT |
-                                 VERBOSE_MIRROR_ID
+                                 VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE
 };
 /* Compatibility with original names */
 #define VERBOSE_SIZE   VERBOSE_STRIPE_SIZE
@@ -212,7 +213,8 @@ struct find_param {
                                 fp_mirror_index_sign:2,
                                 fp_mirror_id_sign:2,
                                 fp_mdt_count_sign:2,
-                                fp_blocks_sign:2;
+                                fp_blocks_sign:2,
+                                fp_ext_size_sign:2;
        unsigned long long       fp_size;
        unsigned long long       fp_size_units;
 
@@ -269,7 +271,9 @@ struct find_param {
                                 fp_check_blocks:1,
                                 fp_exclude_blocks:1,
                                 fp_check_foreign:1,
-                                fp_exclude_foreign:1;
+                                fp_exclude_foreign:1,
+                                fp_check_ext_size:1, /* extension size */
+                                fp_exclude_ext_size:1;
 
        enum llapi_layout_verbose fp_verbose;
        int                      fp_quiet;
@@ -320,6 +324,8 @@ struct find_param {
        unsigned                 fp_projid;
        unsigned long long       fp_blocks;
        unsigned long long       fp_blocks_units;
+       unsigned long long       fp_ext_size;
+       unsigned long long       fp_ext_size_units;
 
        /* In-process parameters. */
        unsigned long            fp_got_uuids:1,

diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh
index dfdd83f..714d6cb 100644 (file)
--- a/lustre/tests/sanity-pfl.sh
+++ b/lustre/tests/sanity-pfl.sh
@@ -974,7 +974,52 @@ test19_io_base() {
 
        flg_opts="--comp-flags extension"
        found=$($LFS find --comp-start 128M -E EOF $flg_opts $comp_file | wc -l)
-       [ $found -eq 1 ] || error "Write: Second component not found"
+       [ $found -eq 1 ] || error "Write: second component not found"
+
+       found=$($LFS find -z 64M $comp_file | wc -l)
+       [ $found -eq 1 ] || error "component not found by ext size"
+
+       found=$($LFS find --extension-size +63M $comp_file | wc -l)
+       [ $found -eq 1 ] || error "component not found by +ext size"
+
+       found=$($LFS find --ext-size -65M $comp_file | wc -l)
+       [ $found -eq 1 ] || error "component not found by -ext size"
+
+       found=$($LFS find -z 65M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by wrong ext size"
+
+       found=$($LFS find -z +65M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by wrong +ext size"
+
+       found=$($LFS find -z -63M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by wrong -ext size"
+
+       found=$($LFS find ! -z 64M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by negation of ext size"
+
+       found=$($LFS find ! -z +63M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by negation of +ext size"
+
+       found=$($LFS find ! -z -65M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by negation of -ext size"
+
+       found=$($LFS find ! -z 65M $comp_file | wc -l)
+       [ $found -eq 1 ] ||
+               error "component not found by negation of wrong ext size"
+
+       found=$($LFS find ! -z +65M $comp_file | wc -l)
+       [ $found -eq 1 ] ||
+               error "component not found by negation of wrong +ext size"
+
+       found=$($LFS find ! -z -63M $comp_file | wc -l)
+       [ $found -eq 1 ] ||
+               error "component not found by negation of wrong -ext size"
+
+       found=$($LFS find -S +1M $comp_file | wc -l)
+       [ $found -eq 0 ] || error "component found by wrong +stripe size"
+
+       found=$($LFS find -c 1 $comp_file | wc -l)
+       [ $found -eq 1 ] || error "component not found by stripe count"
 
        small_write $comp_file $rw_len || error "Verify RW failed"
 

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 67595de..33d09d8 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -405,6 +405,7 @@ command_t cmdlist[] = {
         "                 [--component-id[=comp_id]|-I[comp_id]]\n"
         "                 [--component-flags[=comp_flags]]\n"
         "                 [--component-count]\n"
+        "                 [--extension-size|--ext-size|-z]\n"
         "                 [--component-start[=[+-]comp_start]]\n"
         "                 [--component-end[=[+-]comp_end]|-E[[+-]comp_end]]\n"
         "                 [[!] --mirror-index=[+-]<index> |\n"
@@ -2809,6 +2810,7 @@ static int lfs_setstripe_internal(int argc, char **argv,
        while ((c = getopt_long(argc, argv,
                                "bc:C:dDE:f:H:i:I:m:N::no:p:L:s:S:vx:y:z:",
                                long_opts, NULL)) >= 0) {
+               size_units = 1;
                switch (c) {
                case 0:
                        /* Long options. */
@@ -3837,6 +3839,9 @@ static int lfs_find(int argc, char **argv)
        { .val = 'T',   .name = "mdt-count",    .has_arg = required_argument },
        { .val = 'u',   .name = "uid",          .has_arg = required_argument },
        { .val = 'U',   .name = "user",         .has_arg = required_argument },
+       { .val = 'z',   .name = "extension-size",
+                                               .has_arg = required_argument },
+       { .val = 'z',   .name = "ext-size",     .has_arg = required_argument },
 /* getstripe { .val = 'v', .name = "verbose",  .has_arg = no_argument }, */
 /* getstripe { .val = 'y', .name = "yaml",     .has_arg = no_argument }, */
        { .name = NULL } };
@@ -3852,7 +3857,7 @@ static int lfs_find(int argc, char **argv)
 
        /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */
        while ((c = getopt_long_only(argc, argv,
-                       "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:v",
+                       "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:vz:",
                        long_opts, NULL)) >= 0) {
                 xtime = NULL;
                 xsign = NULL;
@@ -4356,6 +4361,28 @@ err_free:
                        param.fp_check_mdt_count = 1;
                        param.fp_exclude_mdt_count = !!neg_opt;
                        break;
+               case 'z':
+                       if (optarg[0] == '+') {
+                               param.fp_ext_size_sign = -1;
+                               optarg++;
+                       } else if (optarg[0] == '-') {
+                               param.fp_ext_size_sign =  1;
+                               optarg++;
+                       }
+
+                       ret = llapi_parse_size(optarg, &param.fp_ext_size,
+                                              &param.fp_ext_size_units, 0);
+                       if (ret) {
+                               fprintf(stderr, "error: bad ext-size '%s'\n",
+                                       optarg);
+                               goto err;
+                       }
+                       param.fp_ext_size /= SEL_UNIT_SIZE;
+                       param.fp_ext_size_units /= SEL_UNIT_SIZE;
+                       param.fp_check_ext_size = 1;
+                       param.fp_exclude_ext_size = !!neg_opt;
+                       break;
+
                 default:
                         ret = CMD_HELP;
                         goto err;
@@ -4450,6 +4477,8 @@ static int lfs_getstripe_internal(int argc, char **argv,
 /* find        { .val = 'U',   .name = "user",         .has_arg = required_argument }*/
        { .val = 'v',   .name = "verbose",      .has_arg = no_argument },
        { .val = 'y',   .name = "yaml",         .has_arg = no_argument },
+       { .val = 'z',   .name = "extension-size", .has_arg = no_argument },
+       { .val = 'z',   .name = "ext-size",     .has_arg = no_argument },
        { .name = NULL } };
        int c, rc;
        int neg_opt = 0;
@@ -4458,7 +4487,7 @@ static int lfs_getstripe_internal(int argc, char **argv,
        char *end, *tmp;
 
        while ((c = getopt_long(argc, argv,
-                       "-cdDE::FghiI::LmMNoO:pqrRsSvy",
+                       "-cdDE::FghiI::LmMNoO:pqrRsSvyz",
                        long_opts, NULL)) != -1) {
                if (neg_opt)
                        --neg_opt;
@@ -4718,6 +4747,12 @@ static int lfs_getstripe_internal(int argc, char **argv,
                case 'y':
                        param->fp_yaml = 1;
                        break;
+               case 'z':
+                       if (!(param->fp_verbose & VERBOSE_DETAIL)) {
+                               param->fp_verbose |= VERBOSE_EXT_SIZE;
+                               param->fp_max_depth = 0;
+                       }
+                       break;
                default:
                        return CMD_HELP;
                }

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 4065574..f81effb 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -2741,12 +2741,13 @@ static void lov_dump_user_lmm_header(struct lov_user_md *lum, char *path,
                        separator = "\n";
        }
 
-       if (verbose & VERBOSE_STRIPE_SIZE) {
+       if (((verbose & VERBOSE_STRIPE_SIZE) && !extension) ||
+           ((verbose & VERBOSE_EXT_SIZE) && extension)) {
                llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
-               if (verbose & ~VERBOSE_STRIPE_SIZE && extension)
+               if (verbose & ~VERBOSE_EXT_SIZE && extension)
                        llapi_printf(LLAPI_MSG_NORMAL, "%s%sextension_size: ",
                                     space, prefix);
-               else if (verbose & ~VERBOSE_STRIPE_SIZE)
+               if (verbose & ~VERBOSE_STRIPE_SIZE && !extension)
                        llapi_printf(LLAPI_MSG_NORMAL, "%s%sstripe_size:   ",
                                     space, prefix);
                if (is_dir && !is_raw && lum->lmm_stripe_size == 0) {
@@ -3498,8 +3499,15 @@ static void lov_dump_comp_v1(struct find_param *param, char *path,
                         */
                        if (entry->lcme_flags & LCME_FL_INIT)
                                continue;
-                       else
-                               break;
+
+                       if (param->fp_verbose & VERBOSE_EXT_SIZE) {
+                               if (entry->lcme_flags & LCME_FL_EXTENSION)
+                                       /* moved back below */
+                                       i++;
+                               else
+                                       continue;
+                       }
+                       break;
                }
 
                if (entry->lcme_flags & LCME_FL_INIT) {
@@ -4019,16 +4027,23 @@ static int find_check_stripe_size(struct find_param *param)
        if (v1->lmm_magic == LOV_USER_MAGIC_FOREIGN)
                return param->fp_exclude_stripe_size ? 1 : -1;
 
+       ret = param->fp_exclude_stripe_size ? 1 : -1;
        if (v1->lmm_magic == LOV_USER_MAGIC_COMP_V1) {
                comp_v1 = (struct lov_comp_md_v1 *)v1;
                count = comp_v1->lcm_entry_count;
-               ret = param->fp_exclude_stripe_size ? 1 : -1;
        }
 
        for (i = 0; i < count; i++) {
-               if (comp_v1)
+               struct lov_comp_md_entry_v1 *ent;
+
+               if (comp_v1) {
                        v1 = lov_comp_entry(comp_v1, i);
 
+                       ent = &comp_v1->lcm_entries[i];
+                       if (ent->lcme_flags & LCME_FL_EXTENSION)
+                               continue;
+               }
+
                ret = find_value_cmp(v1->lmm_stripe_size, param->fp_stripe_size,
                                     param->fp_stripe_size_sign,
                                     param->fp_exclude_stripe_size,
@@ -4041,6 +4056,38 @@ static int find_check_stripe_size(struct find_param *param)
        return ret;
 }
 
+static int find_check_ext_size(struct find_param *param)
+{
+       struct lov_comp_md_v1 *comp_v1;
+       struct lov_user_md_v1 *v1;
+       int ret, i;
+
+       ret = param->fp_exclude_ext_size ? 1 : -1;
+       comp_v1 = (struct lov_comp_md_v1 *)&param->fp_lmd->lmd_lmm;
+       if (comp_v1->lcm_magic != LOV_USER_MAGIC_COMP_V1)
+               return ret;
+
+       for (i = 0; i < comp_v1->lcm_entry_count; i++) {
+               struct lov_comp_md_entry_v1 *ent;
+
+               v1 = lov_comp_entry(comp_v1, i);
+
+               ent = &comp_v1->lcm_entries[i];
+               if (!(ent->lcme_flags & LCME_FL_EXTENSION))
+                       continue;
+
+               ret = find_value_cmp(v1->lmm_stripe_size, param->fp_ext_size,
+                                    param->fp_ext_size_sign,
+                                    param->fp_exclude_ext_size,
+                                    param->fp_ext_size_units, 0);
+               /* If any ext_size matches */
+               if (ret != -1)
+                       break;
+       }
+
+       return ret;
+}
+
 static __u32 find_get_stripe_count(struct find_param *param)
 {
        struct lov_comp_md_v1 *comp_v1 = NULL;
@@ -4057,8 +4104,15 @@ static __u32 find_get_stripe_count(struct find_param *param)
        }
 
        for (i = 0; i < count; i++) {
-               if (comp_v1)
+               if (comp_v1) {
+                       struct lov_comp_md_entry_v1 *ent;
+
                        v1 = lov_comp_entry(comp_v1, i);
+
+                       ent = &comp_v1->lcm_entries[i];
+                       if (ent->lcme_flags & LCME_FL_EXTENSION)
+                               continue;
+               }
                stripe_count += v1->lmm_stripe_count;
        }
 
@@ -4308,7 +4362,7 @@ static bool find_check_lmm_info(struct find_param *param)
               param->fp_check_comp_count || param->fp_check_comp_end ||
               param->fp_check_comp_start || param->fp_check_comp_flags ||
               param->fp_check_mirror_count || param->fp_check_foreign ||
-              param->fp_check_mirror_state ||
+              param->fp_check_mirror_state || param->fp_check_ext_size ||
               param->fp_check_projid;
 }
 
@@ -4506,6 +4560,12 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp,
                        goto decided;
        }
 
+       if (param->fp_check_ext_size) {
+               decision = find_check_ext_size(param);
+               if (decision == -1)
+                       goto decided;
+       }
+
        if (param->fp_check_stripe_count) {
                decision = find_value_cmp(stripe_count, param->fp_stripe_count,
                                          param->fp_stripe_count_sign,