From c7cf7a5076440f68dee8f7798f46f50c94404a7e Mon Sep 17 00:00:00 2001 From: Vitaly Fertman Date: Mon, 3 Jun 2019 19:34:05 +0300 Subject: [PATCH] LU-10070 utils: SEL: lfs find & getstripe support The support includes: - add --extension-size option to lfs find & getstripe along with +/- functionality; - do not take the extension components into account for lfs find --stripe-size and --stripe-count; - add appropriate tests; Cray-bug-id: LUS-2528 Signed-off-by: Vitaly Fertman Change-Id: Ic3ad4c713e8c676998cf7d02b524ba266c992924 Reviewed-on: https://review.whamcloud.com/34909 Reviewed-by: Patrick Farrell Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexey Lyashkov Reviewed-by: Oleg Drokin --- lustre/doc/lfs-find.1 | 9 ++++- lustre/doc/lfs-getstripe.1 | 12 ++++-- lustre/include/lustre/lustreapi.h | 12 ++++-- lustre/tests/sanity-pfl.sh | 47 ++++++++++++++++++++++- lustre/utils/lfs.c | 39 +++++++++++++++++++- lustre/utils/liblustreapi.c | 78 ++++++++++++++++++++++++++++++++++----- 6 files changed, 178 insertions(+), 19 deletions(-) diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index f44a366..92f879b 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -10,6 +10,7 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--component-end|\fB--comp-end\fR|\fB-E\fR [\fB+-\fR]\fIn\fR[\fBKMGTPE\fR]] [[\fB!\fR] \fB--component-flags|\fB--comp-flags\fR <[^]\fIflag\fB,\fR...>] [[\fB!\fR] \fB--component-start|\fB--comp-start\fR [\fB+-\fR]\fIn\fR[\fBKMGTPE\fR]] + [[\fB!\fR] \fB--extension-size|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]] [[\fB!\fR] \fB--gid\fR|\fB-g\fR|\fB--group\fR|\fB-G\fR <\fIgname\fR>|<\fIgid\fR>] [[\fB!\fR] \fB--layout\fR|\fB-L mdt\fR,\fBraid0\fR,\fBreleased\fR] [[\fB!\fR] \fB--foreign\fR [<\fItype\fR>]] @@ -244,7 +245,13 @@ are multiple OSTs that are being replaced. Stripe size is \fIn\fR bytes, or \fBK\fRibi-, \fBM\fRebi-, \fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-abytes if a suffix is given. For composite files, this matches the stripe -size of the last initialized component. +size of any non-extension component. +.TP +.BR --extension-size | --ext-size | -z +Extension size is \fIn\fR bytes, or \fBK\fRibi-, \fBM\fRebi-, +\fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-abytes if a +suffix is given. For composite files, this matches the extension +size of any extension component. .TP .BR --type | -t File has type: \fBb\fRlock, \fBc\fRharacter, \fBd\fRirectory, diff --git a/lustre/doc/lfs-getstripe.1 b/lustre/doc/lfs-getstripe.1 index 7c4190b..9321c69 100644 --- a/lustre/doc/lfs-getstripe.1 +++ b/lustre/doc/lfs-getstripe.1 @@ -8,6 +8,7 @@ lfs getstripe \- Lustre client command to print layout parameters of a file [\fB--component-flags\fR|\fB--comp-flags\fR[\fB=\fIflags\fR]] [\fB--component-id\fR|\fB--comp-id\fR[=\fIid\fR]|\fB-I\fR[\fIid\fR]] [\fB--component-start\fR[\fB=\fR[\fB+-\fR]\fIstart\fR[\fBKMGTPE\fR]]] + [\fB--extension-size\fR|\fB--ext-size\fR|\fB-z\fR] [\fB--directory\fR|\fB-d\fR] [\fB--fid\fR|\fB-F\fR] [\fB--generation\fR|\fB-g\fR] @@ -21,10 +22,10 @@ lfs getstripe \- Lustre client command to print layout parameters of a file [\fB--stripe-count\fR|\fB-c\fR] [\fB--stripe-index\fR|\fB-i\fR] [\fB--stripe-size\fR|\fB-S\fR] -[[\fB!\fR] \fB--mirror-index\fR=[\fB+-\fR]\fI\fR | [\fB!\fR] \fB--mirror-id\fR=[\fB+-\fR]\fI\fR] -[\fB--verbose\fR|\fB-v\fR] + [[\fB!\fR] \fB--mirror-index\fR=[\fB+-\fR]\fI\fR | [\fB!\fR] \fB--mirror-id\fR=[\fB+-\fR]\fI\fR] + [\fB--verbose\fR|\fB-v\fR] [\fB--yaml\fR|\fB-y\fR] - <\fIdirname\fR|\fIfilename\fR> ... +<\fIdirname\fR|\fIfilename\fR> ... .SH DESCRIPTION .B lfs getstripe is used to list the layout/striping information for a given filename or @@ -33,6 +34,7 @@ and allocated OST objects for each file will be shown. If you only want specific layout information to be printed, then the .BR --stripe-count , .BR --stripe-size , +.BR --extension-size , .BR --stripe-index , .BR --layout , .BR --fid , @@ -235,6 +237,10 @@ Print the starting OST index for the file layout. Print the stripe size in bytes. For composite files this is the stripe size of the last initialized component. .TP +.BR --extension-size | --ext-size | -z +Print the extension size in bytes. For composite files this is the extension +size of the first extension component. +.TP .BR --verbose | -v Also print the layout magic, FID sequence, FID object ID, and FID, in addition to the normally-printed attributes. diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 6ae3e66..97d10bb 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -172,6 +172,7 @@ enum llapi_layout_verbose { VERBOSE_HASH_TYPE = 0x8000, VERBOSE_MIRROR_COUNT = 0x10000, VERBOSE_MIRROR_ID = 0x20000, + VERBOSE_EXT_SIZE = 0x40000, VERBOSE_DEFAULT = VERBOSE_STRIPE_COUNT | VERBOSE_STRIPE_SIZE | VERBOSE_STRIPE_OFFSET | VERBOSE_POOL | VERBOSE_OBJID | VERBOSE_GENERATION | @@ -179,7 +180,7 @@ enum llapi_layout_verbose { VERBOSE_COMP_COUNT | VERBOSE_COMP_FLAGS | VERBOSE_COMP_START | VERBOSE_COMP_END | VERBOSE_COMP_ID | VERBOSE_MIRROR_COUNT | - VERBOSE_MIRROR_ID + VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE }; /* Compatibility with original names */ #define VERBOSE_SIZE VERBOSE_STRIPE_SIZE @@ -212,7 +213,8 @@ struct find_param { fp_mirror_index_sign:2, fp_mirror_id_sign:2, fp_mdt_count_sign:2, - fp_blocks_sign:2; + fp_blocks_sign:2, + fp_ext_size_sign:2; unsigned long long fp_size; unsigned long long fp_size_units; @@ -269,7 +271,9 @@ struct find_param { fp_check_blocks:1, fp_exclude_blocks:1, fp_check_foreign:1, - fp_exclude_foreign:1; + fp_exclude_foreign:1, + fp_check_ext_size:1, /* extension size */ + fp_exclude_ext_size:1; enum llapi_layout_verbose fp_verbose; int fp_quiet; @@ -320,6 +324,8 @@ struct find_param { unsigned fp_projid; unsigned long long fp_blocks; unsigned long long fp_blocks_units; + unsigned long long fp_ext_size; + unsigned long long fp_ext_size_units; /* In-process parameters. */ unsigned long fp_got_uuids:1, diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index dfdd83f..714d6cb 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -974,7 +974,52 @@ test19_io_base() { flg_opts="--comp-flags extension" found=$($LFS find --comp-start 128M -E EOF $flg_opts $comp_file | wc -l) - [ $found -eq 1 ] || error "Write: Second component not found" + [ $found -eq 1 ] || error "Write: second component not found" + + found=$($LFS find -z 64M $comp_file | wc -l) + [ $found -eq 1 ] || error "component not found by ext size" + + found=$($LFS find --extension-size +63M $comp_file | wc -l) + [ $found -eq 1 ] || error "component not found by +ext size" + + found=$($LFS find --ext-size -65M $comp_file | wc -l) + [ $found -eq 1 ] || error "component not found by -ext size" + + found=$($LFS find -z 65M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by wrong ext size" + + found=$($LFS find -z +65M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by wrong +ext size" + + found=$($LFS find -z -63M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by wrong -ext size" + + found=$($LFS find ! -z 64M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by negation of ext size" + + found=$($LFS find ! -z +63M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by negation of +ext size" + + found=$($LFS find ! -z -65M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by negation of -ext size" + + found=$($LFS find ! -z 65M $comp_file | wc -l) + [ $found -eq 1 ] || + error "component not found by negation of wrong ext size" + + found=$($LFS find ! -z +65M $comp_file | wc -l) + [ $found -eq 1 ] || + error "component not found by negation of wrong +ext size" + + found=$($LFS find ! -z -63M $comp_file | wc -l) + [ $found -eq 1 ] || + error "component not found by negation of wrong -ext size" + + found=$($LFS find -S +1M $comp_file | wc -l) + [ $found -eq 0 ] || error "component found by wrong +stripe size" + + found=$($LFS find -c 1 $comp_file | wc -l) + [ $found -eq 1 ] || error "component not found by stripe count" small_write $comp_file $rw_len || error "Verify RW failed" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 67595de..33d09d8 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -405,6 +405,7 @@ command_t cmdlist[] = { " [--component-id[=comp_id]|-I[comp_id]]\n" " [--component-flags[=comp_flags]]\n" " [--component-count]\n" + " [--extension-size|--ext-size|-z]\n" " [--component-start[=[+-]comp_start]]\n" " [--component-end[=[+-]comp_end]|-E[[+-]comp_end]]\n" " [[!] --mirror-index=[+-] |\n" @@ -2809,6 +2810,7 @@ static int lfs_setstripe_internal(int argc, char **argv, while ((c = getopt_long(argc, argv, "bc:C:dDE:f:H:i:I:m:N::no:p:L:s:S:vx:y:z:", long_opts, NULL)) >= 0) { + size_units = 1; switch (c) { case 0: /* Long options. */ @@ -3837,6 +3839,9 @@ static int lfs_find(int argc, char **argv) { .val = 'T', .name = "mdt-count", .has_arg = required_argument }, { .val = 'u', .name = "uid", .has_arg = required_argument }, { .val = 'U', .name = "user", .has_arg = required_argument }, + { .val = 'z', .name = "extension-size", + .has_arg = required_argument }, + { .val = 'z', .name = "ext-size", .has_arg = required_argument }, /* getstripe { .val = 'v', .name = "verbose", .has_arg = no_argument }, */ /* getstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */ { .name = NULL } }; @@ -3852,7 +3857,7 @@ static int lfs_find(int argc, char **argv) /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */ while ((c = getopt_long_only(argc, argv, - "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:v", + "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:vz:", long_opts, NULL)) >= 0) { xtime = NULL; xsign = NULL; @@ -4356,6 +4361,28 @@ err_free: param.fp_check_mdt_count = 1; param.fp_exclude_mdt_count = !!neg_opt; break; + case 'z': + if (optarg[0] == '+') { + param.fp_ext_size_sign = -1; + optarg++; + } else if (optarg[0] == '-') { + param.fp_ext_size_sign = 1; + optarg++; + } + + ret = llapi_parse_size(optarg, ¶m.fp_ext_size, + ¶m.fp_ext_size_units, 0); + if (ret) { + fprintf(stderr, "error: bad ext-size '%s'\n", + optarg); + goto err; + } + param.fp_ext_size /= SEL_UNIT_SIZE; + param.fp_ext_size_units /= SEL_UNIT_SIZE; + param.fp_check_ext_size = 1; + param.fp_exclude_ext_size = !!neg_opt; + break; + default: ret = CMD_HELP; goto err; @@ -4450,6 +4477,8 @@ static int lfs_getstripe_internal(int argc, char **argv, /* find { .val = 'U', .name = "user", .has_arg = required_argument }*/ { .val = 'v', .name = "verbose", .has_arg = no_argument }, { .val = 'y', .name = "yaml", .has_arg = no_argument }, + { .val = 'z', .name = "extension-size", .has_arg = no_argument }, + { .val = 'z', .name = "ext-size", .has_arg = no_argument }, { .name = NULL } }; int c, rc; int neg_opt = 0; @@ -4458,7 +4487,7 @@ static int lfs_getstripe_internal(int argc, char **argv, char *end, *tmp; while ((c = getopt_long(argc, argv, - "-cdDE::FghiI::LmMNoO:pqrRsSvy", + "-cdDE::FghiI::LmMNoO:pqrRsSvyz", long_opts, NULL)) != -1) { if (neg_opt) --neg_opt; @@ -4718,6 +4747,12 @@ static int lfs_getstripe_internal(int argc, char **argv, case 'y': param->fp_yaml = 1; break; + case 'z': + if (!(param->fp_verbose & VERBOSE_DETAIL)) { + param->fp_verbose |= VERBOSE_EXT_SIZE; + param->fp_max_depth = 0; + } + break; default: return CMD_HELP; } diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 4065574..f81effb 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -2741,12 +2741,13 @@ static void lov_dump_user_lmm_header(struct lov_user_md *lum, char *path, separator = "\n"; } - if (verbose & VERBOSE_STRIPE_SIZE) { + if (((verbose & VERBOSE_STRIPE_SIZE) && !extension) || + ((verbose & VERBOSE_EXT_SIZE) && extension)) { llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); - if (verbose & ~VERBOSE_STRIPE_SIZE && extension) + if (verbose & ~VERBOSE_EXT_SIZE && extension) llapi_printf(LLAPI_MSG_NORMAL, "%s%sextension_size: ", space, prefix); - else if (verbose & ~VERBOSE_STRIPE_SIZE) + if (verbose & ~VERBOSE_STRIPE_SIZE && !extension) llapi_printf(LLAPI_MSG_NORMAL, "%s%sstripe_size: ", space, prefix); if (is_dir && !is_raw && lum->lmm_stripe_size == 0) { @@ -3498,8 +3499,15 @@ static void lov_dump_comp_v1(struct find_param *param, char *path, */ if (entry->lcme_flags & LCME_FL_INIT) continue; - else - break; + + if (param->fp_verbose & VERBOSE_EXT_SIZE) { + if (entry->lcme_flags & LCME_FL_EXTENSION) + /* moved back below */ + i++; + else + continue; + } + break; } if (entry->lcme_flags & LCME_FL_INIT) { @@ -4019,16 +4027,23 @@ static int find_check_stripe_size(struct find_param *param) if (v1->lmm_magic == LOV_USER_MAGIC_FOREIGN) return param->fp_exclude_stripe_size ? 1 : -1; + ret = param->fp_exclude_stripe_size ? 1 : -1; if (v1->lmm_magic == LOV_USER_MAGIC_COMP_V1) { comp_v1 = (struct lov_comp_md_v1 *)v1; count = comp_v1->lcm_entry_count; - ret = param->fp_exclude_stripe_size ? 1 : -1; } for (i = 0; i < count; i++) { - if (comp_v1) + struct lov_comp_md_entry_v1 *ent; + + if (comp_v1) { v1 = lov_comp_entry(comp_v1, i); + ent = &comp_v1->lcm_entries[i]; + if (ent->lcme_flags & LCME_FL_EXTENSION) + continue; + } + ret = find_value_cmp(v1->lmm_stripe_size, param->fp_stripe_size, param->fp_stripe_size_sign, param->fp_exclude_stripe_size, @@ -4041,6 +4056,38 @@ static int find_check_stripe_size(struct find_param *param) return ret; } +static int find_check_ext_size(struct find_param *param) +{ + struct lov_comp_md_v1 *comp_v1; + struct lov_user_md_v1 *v1; + int ret, i; + + ret = param->fp_exclude_ext_size ? 1 : -1; + comp_v1 = (struct lov_comp_md_v1 *)¶m->fp_lmd->lmd_lmm; + if (comp_v1->lcm_magic != LOV_USER_MAGIC_COMP_V1) + return ret; + + for (i = 0; i < comp_v1->lcm_entry_count; i++) { + struct lov_comp_md_entry_v1 *ent; + + v1 = lov_comp_entry(comp_v1, i); + + ent = &comp_v1->lcm_entries[i]; + if (!(ent->lcme_flags & LCME_FL_EXTENSION)) + continue; + + ret = find_value_cmp(v1->lmm_stripe_size, param->fp_ext_size, + param->fp_ext_size_sign, + param->fp_exclude_ext_size, + param->fp_ext_size_units, 0); + /* If any ext_size matches */ + if (ret != -1) + break; + } + + return ret; +} + static __u32 find_get_stripe_count(struct find_param *param) { struct lov_comp_md_v1 *comp_v1 = NULL; @@ -4057,8 +4104,15 @@ static __u32 find_get_stripe_count(struct find_param *param) } for (i = 0; i < count; i++) { - if (comp_v1) + if (comp_v1) { + struct lov_comp_md_entry_v1 *ent; + v1 = lov_comp_entry(comp_v1, i); + + ent = &comp_v1->lcm_entries[i]; + if (ent->lcme_flags & LCME_FL_EXTENSION) + continue; + } stripe_count += v1->lmm_stripe_count; } @@ -4308,7 +4362,7 @@ static bool find_check_lmm_info(struct find_param *param) param->fp_check_comp_count || param->fp_check_comp_end || param->fp_check_comp_start || param->fp_check_comp_flags || param->fp_check_mirror_count || param->fp_check_foreign || - param->fp_check_mirror_state || + param->fp_check_mirror_state || param->fp_check_ext_size || param->fp_check_projid; } @@ -4506,6 +4560,12 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, goto decided; } + if (param->fp_check_ext_size) { + decision = find_check_ext_size(param); + if (decision == -1) + goto decided; + } + if (param->fp_check_stripe_count) { decision = find_value_cmp(stripe_count, param->fp_stripe_count, param->fp_stripe_count_sign, -- 1.8.3.1