LU-17699 utils: new --skip option for lfs find

Added [--skip | -k] options for 'lfs find' to skip a percentage of all
files. This brings the benifit of allowing a certain percentage of the
files in the scanned directory to be migraated to new OSTs instead of
migrating entire directory trees. Note that the file size is not taken
into account when skipping files which could result in an unbalanced
set of files being returned. If there are fewer than 25 files there
could be an increased margin of error with the results, however it
should still be relatively negligable (at most 10%).

Planned to implement further utility with --skip-rebalance which would
calculate the pertentage of files that needed to be return vs. skipped
based on the fullness ratio of each OST vs. the avergae fullness of a
balanced filesystem to avoid the user having to calculate the skip
percentage themselves.

Signed-off-by: Frederick Dilger <fdilger@whamcloud.com>
Change-Id: I3ff1600f25f3be54f2a353fa78f7b8b7f98f591a
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55200
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1
index 4fb06c0..52e38e6 100644 (file)
--- a/lustre/doc/lfs-find.1
+++ b/lustre/doc/lfs-find.1
@@ -35,6 +35,7 @@ lfs-find \- Lustre client utility to list files with specific attributes
 [\fB--printf\fR \fIFORMAT\fR]
 [[\fB!\fR] \fB--projid\fR \fIPROJID\fR]
       [[\fB!\fR] \fB--size|\fB-s\fR [\fB-+\fR]\fIn\fR[\fBKMGTPE\fR]]
+      [\fB--skip\fR|\fB-k]\fR \fIPERCENT\fR]
 [[\fB!\fR] \fB--stripe-count|\fB-c\fR [\fB+-\fR]\fIn\fR]
       [[\fB!\fR] \fB--stripe-index|\fB-i\fR \fIn\fR,...]
 [[\fB!\fR] \fB--stripe-size|\fB-S\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]]
@@ -428,6 +429,13 @@ File size is \fIn\fR 512-byte blocks (if no unit is given), or
 \fBG\fRibi-, \fBT\fRebi-, \fBP\fRebi-, or \fBE\fRbi-bytes if a
 suffix is given.
 .TP
+.BR --skip | -k
+Skip \fIPERCENT\fR of the total that match the other filters. This is
+useful together with
+.BR lfs-migrate (1)
+to allow processing only a fraction of the files to rebalance files when new
+OSTs are added to the filesystem.
+.TP
 .BR --stripe-count | -c
 File has \fIn\fR stripes allocated.  For composite files, this
 matches the stripe count of the last initialized component.

diff --git a/lustre/doc/lfs.1 b/lustre/doc/lfs.1
index 2af3c37..7746b82 100644 (file)
--- a/lustre/doc/lfs.1
+++ b/lustre/doc/lfs.1
@@ -37,6 +37,7 @@ lfs \- client utility for Lustre-specific file layout and other attributes
 [\fB--print\fR|\fB-P\fR] [\fB--print0\fR|\fB-0\fR]
       [[\fB!\fR] \fB--projid\fR |<\fIprojid\fR>]
 [[\fB!\fR] \fB--size|\fB-s\fR [\fB-+\fR]\fIn\fR[\fBKMGTPE\fR]]
+      [\fB--skip\fR|\fB-k]\fR \fIn\fR]
       [[\fB!\fR] \fB--stripe-count\fR|\fB-c\fR [\fB+-\fR]\fIn\fR]
       [[\fB!\fR] \fB--stripe-index\fR|\fB-i\fR \fIn\fR,...]
       [[\fB!\fR] \fB--extension-size\fR|\fB-ext-size\fR|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]]

diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index 30d14b1..caf15ae 100644 (file)
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -444,6 +444,9 @@ struct find_param {
        __u64                    fp_attrs;
        __u64                    fp_neg_attrs;
        struct xattr_match_info *fp_xattr_match_info;
+       unsigned long int        fp_skip_percent;
+       unsigned long long       fp_skip_total;
+       unsigned long long       fp_skip_count;
 };
 
 int llapi_ostlist(char *path, struct find_param *param);

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index de7ef96..3445620 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -9174,6 +9174,29 @@ test_56eg() {
 }
 run_test 56eg "lfs find -xattr"
 
+test_56eh() {
+       local dir=$DIR/d$(basetest $testnum)g.$TESTSUITE
+
+       setup_56_special $dir $NUMFILES $NUMDIRS
+
+       local cmd="$LFS find $dir"
+       local total=$($cmd | wc -l)
+       local n=1
+
+       local nums
+
+       echo "Running tests on $dir, with $total total results."
+
+       while (( $n < 100 )); do
+               nums=$($LFS find --skip=$((100 - $n)) $dir | wc -l)
+               (( $nums <= $(($total * ($n + 5) / 100)) &&
+                  $nums >= $(($total * ($n - 5) / 100)) )) ||
+                       error "--skip=$((100 - $n)): expected $(($total * $n / 100)) results with 5% error margin, got $nums"
+               (( n++ ))
+       done
+}
+run_test 56eh "check lfs find --skip"
+
 test_57a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
        # note test will not do anything if MDS is not local

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 92ec916..2b8898c 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -400,7 +400,7 @@ command_t cmdlist[] = {
         "     [[!] --ost|-O <uuid|index,...>] [[!] --perm [/-]mode]\n"
         "     [[!] --pool <pool>] [--print|-P] [--print0|-0] [--printf <format>]\n"
         "     [[!] --projid <projid>] [[!] --size|-s [+-]N[bkMGTPE]]\n"
-        "     [[!] --stripe-count|-c [+-]<stripes>]\n"
+        "     [--skip|-k PERCENT] [[!] --stripe-count|-c [+-]<stripes>]\n"
         "     [[!] --stripe-index|-i <index,...>]\n"
         "     [[!] --stripe-size|-S [+-]N[kMGT]] [[!] --type|-t <filetype>]\n"
         "     [[!] --uid|-u|--user|-U <uid>|<uname>]\n"
@@ -5403,6 +5403,7 @@ static int lfs_find(int argc, char **argv)
        { .val = 'i',   .name = "stripe-index", .has_arg = required_argument },
        { .val = 'i',   .name = "stripe_index", .has_arg = required_argument },
 /* getstripe { .val = 'I', .name = "comp-id",  .has_arg = required_argument }*/
+       { .val = 'k',   .name = "skip",         .has_arg = required_argument },
        { .val = 'l',   .name = "lazy",         .has_arg = no_argument },
        { .val = 'L',   .name = "layout",       .has_arg = required_argument },
        { .val = LFS_LINKS_OPT,
@@ -5459,7 +5460,7 @@ static int lfs_find(int argc, char **argv)
 
        /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */
        while ((c = getopt_long_only(argc, argv,
-               "-0A:b:B:c:C:D:E:g:G:hH:i:lL:m:M:n:N:O:Ppqrs:S:t:T:u:U:z:",
+               "-0A:b:B:c:C:D:E:g:G:hH:i:k:lL:m:M:n:N:O:Ppqrs:S:t:T:u:U:z:",
                long_opts, &optidx)) >= 0) {
                xtime = NULL;
                xsign = NULL;
@@ -5906,6 +5907,18 @@ static int lfs_find(int argc, char **argv)
                                param.fp_check_hash_flag = 1;
                        param.fp_exclude_hash_type = !!neg_opt;
                        break;
+               case 'k':
+                       param.fp_skip_percent = strtoul(optarg, &endptr, 10);
+                       if (param.fp_skip_percent < 0 ||
+                           param.fp_skip_percent >= 100 ||
+                           (*endptr != '\0' && *endptr != '%')) {
+                               ret = -1;
+                               fprintf(stderr,
+                                       "error: invalid skip percentage '%s'\n",
+                                       optarg);
+                               goto err;
+                       }
+                       break;
                case 'l':
                        param.fp_lazy = 1;
                        break;

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 2fedf9b..009b703 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -4843,6 +4843,16 @@ matched:
        return 1;
 }
 
+static bool find_skip_file(struct find_param *param)
+{
+       if (param->fp_skip_count * 100 <
+           param->fp_skip_percent * param->fp_skip_total++) {
+               param->fp_skip_count++;
+               return true;
+       }
+       return false;
+}
+
 static bool find_check_lmm_info(struct find_param *param)
 {
        return param->fp_check_pool || param->fp_check_stripe_count ||
@@ -6077,6 +6087,9 @@ obd_matches:
        }
 
 print:
+       if (param->fp_skip_percent && find_skip_file(param))
+               goto decided;
+
        if (param->fp_format_printf_str)
                printf_format_string(param, path, projid, d);
        else