LU-14361 statahead: increase the initial statahead count

In this patch, we increase the initial stat-ahead count from the
default 8 to 64 during the fname statahead pattern test sanity/
test_123i. The origial starting statahead count is too small, may
result in that the statahead thread quits wrongly. This will fail
sanity/test_123i fairly often.

We also imporve aheadmany and use it to generate the fname stat()
workload to verify that fname statahead pattern works correctly.

Test-Parameters: mdtcount=4 mdscount=2 testlist=sanity env=ONLY=123i,ONLY_REPEAT=100
Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: I7d13120a9480ea5b2e53963789074429c414ff90
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53634
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/aheadmany.c b/lustre/tests/aheadmany.c
index d145858..d01d3fb 100644 (file)
--- a/lustre/tests/aheadmany.c
+++ b/lustre/tests/aheadmany.c
@@ -59,6 +59,7 @@ static void usage(void)
               "\t--start|-s:   Start index of file names\n"
               "\t--end|-e:     End index of file names\n"
               "\t--basename|-b:Base name for file naming format\n"
+              "\t--noadvise|-N:No ahead advise hint IOCTL\n"
               "\t--batch_max|-B: max batch count for ahead operations\n"
               "\t--directory|-d: under this directory do ahead operations\n",
               progname);
@@ -80,15 +81,40 @@ static char *get_file_name(const char *dirpath, const char *basename, long n)
        return filename;
 }
 
+static int ll_batch_io_by_name(const char *dirpath, const char *fname,
+                              enum lu_access_flags flags, __u64 start,
+                              __u64 end)
+{
+       int rc = 0;
+       int i;
+
+       for (i = start; i < end; i++) {
+               char *filename;
+               struct stat st;
+
+               filename = get_file_name(dirpath, fname, i);
+               if (flags & ACCESS_FL_STAT) {
+                       rc = stat(filename, &st);
+                       if (rc < 0) {
+                               rc = -errno;
+                               fprintf(stderr,
+                                       "%s: stat(%s) failed: rc = %d\n",
+                                       progname, filename, errno);
+                               break;
+                       }
+               }
+       }
+
+       return rc;
+}
+
 static int ll_ahead_by_name_index(const char *dirpath, const char *fname,
                                  enum lu_access_flags flags, __u64 start,
                                  __u64 end, __u32 batch_max)
 {
        struct llapi_lu_ladvise2 ladvise;
-       struct stat st;
        int dir_fd;
        int rc;
-       int i;
 
        dir_fd = open(dirpath, O_DIRECTORY | O_RDONLY);
        if (dir_fd < 0) {
@@ -113,19 +139,7 @@ static int ll_ahead_by_name_index(const char *dirpath, const char *fname,
                return rc;
        }
 
-       for (i = start; i < end; i++) {
-               char *filename;
-
-               filename = get_file_name(dirpath, fname, i);
-               rc = stat(filename, &st);
-               if (rc < 0) {
-                       rc = -errno;
-                       fprintf(stderr, "%s: stat(%s) failed: rc = %d\n",
-                               progname, filename, errno);
-                       break;
-               }
-       }
-
+       rc = ll_batch_io_by_name(dirpath, fname, flags, start, end);
        close(dir_fd);
        return rc;
 }
@@ -139,6 +153,7 @@ int main(int argc, char **argv)
        { .val = 'b',   .name = "basename",     .has_arg = required_argument },
        { .val = 'd',   .name = "directory",    .has_arg = required_argument },
        { .val = 'B',   .name = "batch_max",    .has_arg = required_argument },
+       { .val = 'N',   .name = "noadvise",     .has_arg = no_argument },
        { .val = 'h',   .name = "help",         .has_arg = no_argument },
        { .name = NULL } };
        enum lu_access_flags flags = ACCESS_FL_NONE;
@@ -148,12 +163,13 @@ int main(int argc, char **argv)
        char *fname = NULL;
        __u64 start_index = 0;
        __u64 end_index = 0;
+       bool has_advise = true;
        char *end;
        int rc = 0;
        int c;
 
        progname = basename(argv[0]);
-       while ((c = getopt_long(argc, argv, "c:s:e:b:d:B:h",
+       while ((c = getopt_long(argc, argv, "c:s:e:b:d:B:Nh",
                                long_opts, NULL)) != -1) {
                switch (c) {
                case 'c':
@@ -198,6 +214,9 @@ int main(int argc, char **argv)
                                return -EINVAL;
                        }
                        break;
+               case 'N':
+                       has_advise = false;
+                       break;
                case 'b':
                        fname = optarg;
                        break;
@@ -242,8 +261,13 @@ int main(int argc, char **argv)
                        return -EINVAL;
                }
 
-               rc = ll_ahead_by_name_index(dirpath, fname, flags, start_index,
-                                           end_index, batch_max);
+               if (has_advise)
+                       rc = ll_ahead_by_name_index(dirpath, fname, flags,
+                                                   start_index, end_index,
+                                                   batch_max);
+               else
+                       rc = ll_batch_io_by_name(dirpath, fname, flags,
+                                                start_index, end_index);
        } else {
                rc = -EOPNOTSUPP;
                fprintf(stderr, "%s: unsupported ahead type %d\n",

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 71deb1d..e2ef8cb 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -14605,11 +14605,16 @@ test_123h() {
 }
 run_test 123h "Verify statahead work with the fname pattern via du"
 
-test_123i() {
+test_123i_base() {
+       local fmt=$1
+       local iocmd=$2
        local dir=$DIR/$tdir
-       local cmd="createmany -m $dir/$tfile.%06d 1000"
+       local cmd="createmany -m $fmt"
 
-       stack_trap "unlinkmany $dir/$tfile.%06d 1000"
+       echo "Command:"
+       echo "- $cmd"
+       echo "- $iocmd"
+       stack_trap "unlinkmany $fmt"
        mkdir -p $dir || error "failed to mkdir $dir"
        eval $cmd
 
@@ -14617,23 +14622,9 @@ test_123i() {
        $LCTL set_param llite.*.statahead_stats=clear
        $LCTL set_param mdc.*.batch_stats=0
 
-       local max
-       local batch_max
-       local enabled
-
-       max=$($LCTL get_param -n llite.*.statahead_max | head -n 1)
-       batch_max=$($LCTL get_param -n llite.*.statahead_batch_max | head -n 1)
-       enabled=$($LCTL get_param -n llite.*.enable_statahead_fname | head -n 1)
-       stack_trap "$LCTL set_param llite.*.statahead_max=$max"
-       stack_trap "$LCTL set_param llite.*.statahead_batch_max=$batch_max"
-       stack_trap "$LCTL set_param llite.*.enable_statahead_fname=$enabled"
-
-       $LCTL set_param llite.*.statahead_max=1024
-       $LCTL set_param llite.*.statahead_batch_max=32
-       $LCTL set_param llite.*.enable_statahead_fname=1
        echo "statahead_stats (Pre):"
        lctl get_param -n llite.*.statahead_stats
-       ls $dir/* > /dev/null
+       eval $iocmd || error "$iocmd failed"
        echo "statahead_stats (Post):"
        $LCTL get_param -n llite.*.statahead_stats
        $LCTL get_param -n mdc.*.batch_stats
@@ -14651,9 +14642,35 @@ test_123i() {
        count=$($LCTL get_param -n llite.*.statahead_stats |
                awk '/hit.total:/ {print $2}')
        # Hit ratio should be >= 75%
-       (( $count > 75 )) || error "hit total is too low: $count"
+       (( $count > 750 )) || error "hit total is too low: $count"
+}
+
+test_123i() {
+       local dir=$DIR/$tdir
+       local cnt=1000
+       local max
+       local batch_max
+       local enabled
+       local min
+
+       max=$($LCTL get_param -n llite.*.statahead_max | head -n 1)
+       batch_max=$($LCTL get_param -n llite.*.statahead_batch_max | head -n 1)
+       min=$($LCTL get_param -n llite.*.statahead_min | head -n 1)
+       enabled=$($LCTL get_param -n llite.*.enable_statahead_fname | head -n 1)
+       stack_trap "$LCTL set_param llite.*.statahead_max=$max"
+       stack_trap "$LCTL set_param llite.*.statahead_batch_max=$batch_max"
+       stack_trap "$LCTL set_param llite.*.statahead_min=$min"
+       stack_trap "$LCTL set_param llite.*.enable_statahead_fname=$enabled"
+       $LCTL set_param llite.*.statahead_max=1024
+       $LCTL set_param llite.*.statahead_batch_max=32
+       $LCTL set_param llite.*.statahead_min=64
+       $LCTL set_param llite.*.enable_statahead_fname=1
+
+       test_123i_base "$dir/$tfile.%06d $cnt" "ls $dir/* > /dev/null"
+       test_123i_base "$dir/$tfile $cnt" \
+               "aheadmany -c stat -N -s 0 -e $cnt -b $tfile -d $dir"
 }
-run_test 123i "Verify statahead work with the fname pattern via ls dir/*"
+run_test 123i "Verify statahead work with the fname indexing pattern"
 
 test_124a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"