From: Frederick Dilger Date: Sat, 25 May 2024 21:46:08 +0000 (-0400) Subject: LU-17699 utils: new --skip option for lfs find X-Git-Tag: 2.15.64~119 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F00%2F55200%2F5;p=fs%2Flustre-release.git LU-17699 utils: new --skip option for lfs find Added [--skip | -k] options for 'lfs find' to skip a percentage of all files. This brings the benifit of allowing a certain percentage of the files in the scanned directory to be migraated to new OSTs instead of migrating entire directory trees. Note that the file size is not taken into account when skipping files which could result in an unbalanced set of files being returned. If there are fewer than 25 files there could be an increased margin of error with the results, however it should still be relatively negligable (at most 10%). Planned to implement further utility with --skip-rebalance which would calculate the pertentage of files that needed to be return vs. skipped based on the fullness ratio of each OST vs. the avergae fullness of a balanced filesystem to avoid the user having to calculate the skip percentage themselves. Signed-off-by: Frederick Dilger Change-Id: I3ff1600f25f3be54f2a353fa78f7b8b7f98f591a Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55200 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 4fb06c0..52e38e6 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -35,6 +35,7 @@ lfs-find \- Lustre client utility to list files with specific attributes [\fB--printf\fR \fIFORMAT\fR] [[\fB!\fR] \fB--projid\fR \fIPROJID\fR] [[\fB!\fR] \fB--size|\fB-s\fR [\fB-+\fR]\fIn\fR[\fBKMGTPE\fR]] + [\fB--skip\fR|\fB-k]\fR \fIPERCENT\fR] [[\fB!\fR] \fB--stripe-count|\fB-c\fR [\fB+-\fR]\fIn\fR] [[\fB!\fR] \fB--stripe-index|\fB-i\fR \fIn\fR,...] [[\fB!\fR] \fB--stripe-size|\fB-S\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]] @@ -428,6 +429,13 @@ File size is \fIn\fR 512-byte blocks (if no unit is given), or \fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-bytes if a suffix is given. .TP +.BR --skip | -k +Skip \fIPERCENT\fR of the total that match the other filters. This is +useful together with +.BR lfs-migrate (1) +to allow processing only a fraction of the files to rebalance files when new +OSTs are added to the filesystem. +.TP .BR --stripe-count | -c File has \fIn\fR stripes allocated. For composite files, this matches the stripe count of the last initialized component. diff --git a/lustre/doc/lfs.1 b/lustre/doc/lfs.1 index 2af3c37..7746b82 100644 --- a/lustre/doc/lfs.1 +++ b/lustre/doc/lfs.1 @@ -37,6 +37,7 @@ lfs \- client utility for Lustre-specific file layout and other attributes [\fB--print\fR|\fB-P\fR] [\fB--print0\fR|\fB-0\fR] [[\fB!\fR] \fB--projid\fR |<\fIprojid\fR>] [[\fB!\fR] \fB--size|\fB-s\fR [\fB-+\fR]\fIn\fR[\fBKMGTPE\fR]] + [\fB--skip\fR|\fB-k]\fR \fIn\fR] [[\fB!\fR] \fB--stripe-count\fR|\fB-c\fR [\fB+-\fR]\fIn\fR] [[\fB!\fR] \fB--stripe-index\fR|\fB-i\fR \fIn\fR,...] [[\fB!\fR] \fB--extension-size\fR|\fB-ext-size\fR|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]] diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 30d14b1..caf15ae 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -444,6 +444,9 @@ struct find_param { __u64 fp_attrs; __u64 fp_neg_attrs; struct xattr_match_info *fp_xattr_match_info; + unsigned long int fp_skip_percent; + unsigned long long fp_skip_total; + unsigned long long fp_skip_count; }; int llapi_ostlist(char *path, struct find_param *param); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index de7ef96..3445620 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -9174,6 +9174,29 @@ test_56eg() { } run_test 56eg "lfs find -xattr" +test_56eh() { + local dir=$DIR/d$(basetest $testnum)g.$TESTSUITE + + setup_56_special $dir $NUMFILES $NUMDIRS + + local cmd="$LFS find $dir" + local total=$($cmd | wc -l) + local n=1 + + local nums + + echo "Running tests on $dir, with $total total results." + + while (( $n < 100 )); do + nums=$($LFS find --skip=$((100 - $n)) $dir | wc -l) + (( $nums <= $(($total * ($n + 5) / 100)) && + $nums >= $(($total * ($n - 5) / 100)) )) || + error "--skip=$((100 - $n)): expected $(($total * $n / 100)) results with 5% error margin, got $nums" + (( n++ )) + done +} +run_test 56eh "check lfs find --skip" + test_57a() { [ $PARALLEL == "yes" ] && skip "skip parallel run" # note test will not do anything if MDS is not local diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 92ec916..2b8898c 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -400,7 +400,7 @@ command_t cmdlist[] = { " [[!] --ost|-O ] [[!] --perm [/-]mode]\n" " [[!] --pool ] [--print|-P] [--print0|-0] [--printf ]\n" " [[!] --projid ] [[!] --size|-s [+-]N[bkMGTPE]]\n" - " [[!] --stripe-count|-c [+-]]\n" + " [--skip|-k PERCENT] [[!] --stripe-count|-c [+-]]\n" " [[!] --stripe-index|-i ]\n" " [[!] --stripe-size|-S [+-]N[kMGT]] [[!] --type|-t ]\n" " [[!] --uid|-u|--user|-U |]\n" @@ -5403,6 +5403,7 @@ static int lfs_find(int argc, char **argv) { .val = 'i', .name = "stripe-index", .has_arg = required_argument }, { .val = 'i', .name = "stripe_index", .has_arg = required_argument }, /* getstripe { .val = 'I', .name = "comp-id", .has_arg = required_argument }*/ + { .val = 'k', .name = "skip", .has_arg = required_argument }, { .val = 'l', .name = "lazy", .has_arg = no_argument }, { .val = 'L', .name = "layout", .has_arg = required_argument }, { .val = LFS_LINKS_OPT, @@ -5459,7 +5460,7 @@ static int lfs_find(int argc, char **argv) /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */ while ((c = getopt_long_only(argc, argv, - "-0A:b:B:c:C:D:E:g:G:hH:i:lL:m:M:n:N:O:Ppqrs:S:t:T:u:U:z:", + "-0A:b:B:c:C:D:E:g:G:hH:i:k:lL:m:M:n:N:O:Ppqrs:S:t:T:u:U:z:", long_opts, &optidx)) >= 0) { xtime = NULL; xsign = NULL; @@ -5906,6 +5907,18 @@ static int lfs_find(int argc, char **argv) param.fp_check_hash_flag = 1; param.fp_exclude_hash_type = !!neg_opt; break; + case 'k': + param.fp_skip_percent = strtoul(optarg, &endptr, 10); + if (param.fp_skip_percent < 0 || + param.fp_skip_percent >= 100 || + (*endptr != '\0' && *endptr != '%')) { + ret = -1; + fprintf(stderr, + "error: invalid skip percentage '%s'\n", + optarg); + goto err; + } + break; case 'l': param.fp_lazy = 1; break; diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 2fedf9b..009b703 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -4843,6 +4843,16 @@ matched: return 1; } +static bool find_skip_file(struct find_param *param) +{ + if (param->fp_skip_count * 100 < + param->fp_skip_percent * param->fp_skip_total++) { + param->fp_skip_count++; + return true; + } + return false; +} + static bool find_check_lmm_info(struct find_param *param) { return param->fp_check_pool || param->fp_check_stripe_count || @@ -6077,6 +6087,9 @@ obd_matches: } print: + if (param->fp_skip_percent && find_skip_file(param)) + goto decided; + if (param->fp_format_printf_str) printf_format_string(param, path, projid, d); else