LU-575 MRP-133 lfs find speedup

lfs find should send getattr on mds only if needed;
lfs find should not break on matched obd but check other parameters as well;
lfs find time compare fixes;

Reviewed-by: Alexey Lyashkov <alexey_lyashkov@xyratex.com>
Reviewed-by: Colin Faber <colin.faber@xyratex.com>
Signed-off-by: Vitaly Fertman <vitaly_fertman@xyratex.com>
LU-575 MRP-260 fix quota tests

a fix for quota size units which conflicted with lfind size units

Reviewed-by: Alexey Lyashkov <alexey_lyashkov@xyratex.com>
Reviewed-by: Andrew Perepechko <Andrew_Perepechko@xyratex.com>
Signed-off-by: Vitaly Fertman <vitaly_fertman@xyratex.com>
Change-Id: I42b2acee18c6c9e0d787b97a5e957431d2bb4bf1
Reviewed-on: http://review.whamcloud.com/1195
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index c50b040..9aa27b2 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -3408,19 +3408,27 @@ test_56n() {
 run_test 56n "check lfs find -type l ============================="
 
 test_56o() {
 run_test 56n "check lfs find -type l ============================="
 
 test_56o() {

-       setup_56 $NUMFILES $NUMDIRS

        TDIR=$DIR/${tdir}g
        TDIR=$DIR/${tdir}g

+       rm -rf $TDIR
+       setup_56 $NUMFILES $NUMDIRS

 
        utime $TDIR/file1 > /dev/null || error "utime (1)"
        utime $TDIR/file2 > /dev/null || error "utime (2)"
        utime $TDIR/dir1 > /dev/null || error "utime (3)"
        utime $TDIR/dir2 > /dev/null || error "utime (4)"
        utime $TDIR/dir1/file1 > /dev/null || error "utime (5)"
 
        utime $TDIR/file1 > /dev/null || error "utime (1)"
        utime $TDIR/file2 > /dev/null || error "utime (2)"
        utime $TDIR/dir1 > /dev/null || error "utime (3)"
        utime $TDIR/dir2 > /dev/null || error "utime (4)"
        utime $TDIR/dir1/file1 > /dev/null || error "utime (5)"

+       dd if=/dev/zero count=1 >> $TDIR/dir1/file1 && sync
+
+       EXPECTED=4
+       NUMS=`$LFIND -mtime +0 $TDIR | wc -l`
+       [ $NUMS -eq $EXPECTED ] || \
+               error "lfs find -mtime +0 $TDIR wrong: found $NUMS, expected $EXPECTED"

 
 

-       EXPECTED=5
-       NUMS=`$LFIND -mtime +1 $TDIR | wc -l`
+       EXPECTED=12
+       NUMS=`$LFIND -mtime 0 $TDIR | wc -l`

        [ $NUMS -eq $EXPECTED ] || \
        [ $NUMS -eq $EXPECTED ] || \

-               error "lfs find -mtime $TDIR wrong: found $NUMS, expected $EXPECTED"
+               error "lfs find -mtime 0 $TDIR wrong: found $NUMS, expected $EXPECTED"
+

 }
 run_test 56o "check lfs find -mtime for old files =========================="
 
 }
 run_test 56o "check lfs find -mtime for old files =========================="
 


@@ -3483,6 +3491,7 @@ test_56r() {
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR ! -size 0 wrong: found $NUMS, expected $EXPECTED"
        echo "test" > $TDIR/56r && sync
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR ! -size 0 wrong: found $NUMS, expected $EXPECTED"
        echo "test" > $TDIR/56r && sync

+       echo "test2" > $TDIR/56r2 && sync

        EXPECTED=1
        NUMS=`$LFIND -size 5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
        EXPECTED=1
        NUMS=`$LFIND -size 5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \


@@ -3491,14 +3500,18 @@ test_56r() {
        NUMS=`$LFIND -size +5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR -size +5 wrong: found $NUMS, expected $EXPECTED"
        NUMS=`$LFIND -size +5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR -size +5 wrong: found $NUMS, expected $EXPECTED"

-       EXPECTED=13
+       EXPECTED=2

        NUMS=`$LFIND -size +0 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR -size +0 wrong: found $NUMS, expected $EXPECTED"
        NUMS=`$LFIND -size +0 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR -size +0 wrong: found $NUMS, expected $EXPECTED"

-       EXPECTED=0
+       EXPECTED=2

        NUMS=`$LFIND ! -size -5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR ! -size -5 wrong: found $NUMS, expected $EXPECTED"
        NUMS=`$LFIND ! -size -5 -t f $TDIR | wc -l`
        [ $NUMS -eq $EXPECTED ] || \
                error "lfs find $TDIR ! -size -5 wrong: found $NUMS, expected $EXPECTED"

+       EXPECTED=12
+       NUMS=`$LFIND -size -5 -t f $TDIR | wc -l`
+       [ $NUMS -eq $EXPECTED ] || \
+               error "lfs find $TDIR -size -5 wrong: found $NUMS, expected $EXPECTED"

 }
 
 run_test 56r "check lfs find -size works =========================="
 }
 
 run_test 56r "check lfs find -size works =========================="

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 6010105..99707f2 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -277,7 +277,7 @@ static int lfs_setstripe(int argc, char **argv)
         char *stripe_off_arg = NULL;
         char *stripe_count_arg = NULL;
         char *pool_name_arg = NULL;
         char *stripe_off_arg = NULL;
         char *stripe_count_arg = NULL;
         char *pool_name_arg = NULL;

-        unsigned long long size_units;
+        unsigned long long size_units = 1;

 
         struct option long_opts[] = {
                 {"count",       required_argument, 0, 'c'},
 
         struct option long_opts[] = {
                 {"count",       required_argument, 0, 'c'},


@@ -494,7 +494,7 @@ static int lfs_find(int argc, char **argv)
         int new_fashion = 1;
         int c, ret;
         time_t t;
         int new_fashion = 1;
         int c, ret;
         time_t t;

-        struct find_param param = { .maxdepth = -1 };
+        struct find_param param = { .maxdepth = -1, .size_units = 0 };

         struct option long_opts[] = {
                 {"atime",     required_argument, 0, 'A'},
                 {"ctime",     required_argument, 0, 'C'},
         struct option long_opts[] = {
                 {"atime",     required_argument, 0, 'A'},
                 {"ctime",     required_argument, 0, 'C'},


@@ -1565,9 +1565,9 @@ error:
         return ULONG_MAX;
 }
 
         return ULONG_MAX;
 }
 

-#define ARG2ULL(nr, str, defscale)                                      \
+#define ARG2ULL(nr, str, def_units)                                     \

 do {                                                                    \
 do {                                                                    \

-        unsigned long long limit, units = 0;                            \
+        unsigned long long limit, units = def_units;                    \

         int rc;                                                         \
                                                                         \
         rc = parse_size(str, &limit, &units, 1);                        \
         int rc;                                                         \
                                                                         \
         rc = parse_size(str, &limit, &units, 1);                        \


@@ -1575,7 +1575,7 @@ do {                                                                    \
                 fprintf(stderr, "error: bad limit value %s\n", str);    \
                 return CMD_HELP;                                        \
         }                                                               \
                 fprintf(stderr, "error: bad limit value %s\n", str);    \
                 return CMD_HELP;                                        \
         }                                                               \

-        nr = ((units == 0) ? (defscale) : 1) * limit;                   \
+        nr = limit;                                                     \

 } while (0)
 
 static inline int has_times_option(int argc, char **argv)
 } while (0)
 
 static inline int has_times_option(int argc, char **argv)

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index f935ded..e08bef0 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -170,20 +170,22 @@ void llapi_printf(int level, char *fmt, ...)
 }
 
 /**
 }
 
 /**

- * size_units is unchanged if no specifier used
+ * size_units is to be initialized (or zeroed) by caller.

  */
 int parse_size(char *optarg, unsigned long long *size,
                unsigned long long *size_units, int bytes_spec)
 {
         char *end;
 
  */
 int parse_size(char *optarg, unsigned long long *size,
                unsigned long long *size_units, int bytes_spec)
 {
         char *end;
 

+        if (*size_units == 0)
+                *size_units = 1;
+

         *size = strtoull(optarg, &end, 0);
 
         if (*end != '\0') {
                 if ((*end == 'b') && *(end+1) == '\0' &&
                     (*size & (~0ULL << (64 - 9))) == 0 &&
                     !bytes_spec) {
         *size = strtoull(optarg, &end, 0);
 
         if (*end != '\0') {
                 if ((*end == 'b') && *(end+1) == '\0' &&
                     (*size & (~0ULL << (64 - 9))) == 0 &&
                     !bytes_spec) {

-                        *size <<= 9;

                         *size_units = 1 << 9;
                 } else if ((*end == 'b') && *(end+1) == '\0' &&
                            bytes_spec) {
                         *size_units = 1 << 9;
                 } else if ((*end == 'b') && *(end+1) == '\0' &&
                            bytes_spec) {


@@ -191,38 +193,32 @@ int parse_size(char *optarg, unsigned long long *size,
                 } else if ((*end == 'k' || *end == 'K') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 10))) == 0) {
                 } else if ((*end == 'k' || *end == 'K') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 10))) == 0) {

-                        *size <<= 10;

                         *size_units = 1 << 10;
                 } else if ((*end == 'm' || *end == 'M') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 20))) == 0) {
                         *size_units = 1 << 10;
                 } else if ((*end == 'm' || *end == 'M') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 20))) == 0) {

-                        *size <<= 20;

                         *size_units = 1 << 20;
                 } else if ((*end == 'g' || *end == 'G') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 30))) == 0) {
                         *size_units = 1 << 20;
                 } else if ((*end == 'g' || *end == 'G') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 30))) == 0) {

-                        *size <<= 30;

                         *size_units = 1 << 30;
                 } else if ((*end == 't' || *end == 'T') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 40))) == 0) {
                         *size_units = 1 << 30;
                 } else if ((*end == 't' || *end == 'T') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 40))) == 0) {

-                        *size <<= 40;

                         *size_units = 1ULL << 40;
                 } else if ((*end == 'p' || *end == 'P') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 50))) == 0) {
                         *size_units = 1ULL << 40;
                 } else if ((*end == 'p' || *end == 'P') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 50))) == 0) {

-                        *size <<= 50;

                         *size_units = 1ULL << 50;
                 } else if ((*end == 'e' || *end == 'E') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 60))) == 0) {
                         *size_units = 1ULL << 50;
                 } else if ((*end == 'e' || *end == 'E') &&
                            *(end+1) == '\0' && (*size &
                            (~0ULL << (64 - 60))) == 0) {

-                        *size <<= 60;

                         *size_units = 1ULL << 60;
                 } else {
                         return -1;
                 }
         }
                         *size_units = 1ULL << 60;
                 } else {
                         return -1;
                 }
         }

-
+        *size *= *size_units;

         return 0;
 }
 
         return 0;
 }
 


@@ -1854,7 +1850,7 @@ int llapi_file_lookup(int dirfd, const char *name)
  * sign), 1st column is the answer for the MDS value, the 2nd is for the OST:
  * --------------------------------------
  * 1 | file > limit; sign > 0 | -1 / -1 |
  * sign), 1st column is the answer for the MDS value, the 2nd is for the OST:
  * --------------------------------------
  * 1 | file > limit; sign > 0 | -1 / -1 |

- * 2 | file = limit; sign > 0 |  ? /  1 |
+ * 2 | file = limit; sign > 0 | -1 / -1 |

  * 3 | file < limit; sign > 0 |  ? /  1 |
  * 4 | file > limit; sign = 0 | -1 / -1 |
  * 5 | file = limit; sign = 0 |  ? /  1 |  <- (see the Note below)
  * 3 | file < limit; sign > 0 |  ? /  1 |
  * 4 | file > limit; sign = 0 | -1 / -1 |
  * 5 | file = limit; sign = 0 |  ? /  1 |  <- (see the Note below)


@@ -1872,15 +1868,16 @@ static int find_value_cmp(unsigned long long file, unsigned long long limit,
         int ret = -1;
 
         if (sign > 0) {
         int ret = -1;
 
         if (sign > 0) {

-                if (file <= limit)
+                /* Drop the fraction of margin (of days). */
+                if (file + margin <= limit)

                         ret = mds ? 0 : 1;
         } else if (sign == 0) {
                         ret = mds ? 0 : 1;
         } else if (sign == 0) {

-                if (file <= limit && file + margin >= limit)
+                if (file <= limit && file + margin > limit)

                         ret = mds ? 0 : 1;
                 else if (file + margin <= limit)
                         ret = mds ? 0 : -1;
         } else if (sign < 0) {
                         ret = mds ? 0 : 1;
                 else if (file + margin <= limit)
                         ret = mds ? 0 : -1;
         } else if (sign < 0) {

-                if (file >= limit)
+                if (file > limit)

                         ret = 1;
                 else if (mds)
                         ret = 0;
                         ret = 1;
                 else if (mds)
                         ret = 0;


@@ -1899,7 +1896,7 @@ static int find_value_cmp(unsigned long long file, unsigned long long limit,
 static int find_time_check(lstat_t *st, struct find_param *param, int mds)
 {
         int ret;
 static int find_time_check(lstat_t *st, struct find_param *param, int mds)
 {
         int ret;

-        int rc = 0;
+        int rc = 1;

 
         /* Check if file is accepted. */
         if (param->atime) {
 
         /* Check if file is accepted. */
         if (param->atime) {


@@ -1952,7 +1949,8 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
 
         LASSERT(parent != NULL || dir != NULL);
 
 
         LASSERT(parent != NULL || dir != NULL);
 

-        param->lmd->lmd_lmm.lmm_stripe_count = 0;
+        if (param->have_fileinfo == 0)
+                param->lmd->lmd_lmm.lmm_stripe_count = 0;

 
         /* If a regular expression is presented, make the initial decision */
         if (param->pattern != NULL) {
 
         /* If a regular expression is presented, make the initial decision */
         if (param->pattern != NULL) {


@@ -1978,14 +1976,18 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
         }
 
 
         }
 
 

-        /* If a time or OST should be checked, the decision is not taken yet. */
-        if (param->atime || param->ctime || param->mtime || param->obduuid ||
-            param->check_size)
+        ret = 0;
+
+        /* Request MDS for the stat info if some of these parameters need
+         * to be compared. */
+        if (param->obduuid    || param->check_uid || param->check_gid ||
+            param->check_pool || param->atime     || param->ctime     ||
+            param->mtime      || param->check_size)
+                decision = 0;
+        if (param->type && checked_type == 0)

                 decision = 0;
 
                 decision = 0;
 

-        ret = 0;
-        /* Request MDS for the stat info. */
-        if (param->have_fileinfo == 0) {
+        if (param->have_fileinfo == 0 && decision == 0) {

                 if (dir) {
                         /* retrieve needed file info */
                         ret = ioctl(dirfd(dir), LL_IOC_MDC_GETINFO,
                 if (dir) {
                         /* retrieve needed file info */
                         ret = ioctl(dirfd(dir), LL_IOC_MDC_GETINFO,


@@ -2096,15 +2098,17 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
                                             lmm_objects[i].l_ost_idx) {
                                                 if (param->exclude_obd)
                                                         goto decided;
                                             lmm_objects[i].l_ost_idx) {
                                                 if (param->exclude_obd)
                                                         goto decided;

-                                                goto obd_matches;
+                                                break;

                                         }
                                 }
                                         }
                                 }

+                                /* If an OBD matches, just break */
+                                if (j != param->num_obds)
+                                        break;

                         }
 
                         if (i == param->lmd->lmd_lmm.lmm_stripe_count) {
                         }
 
                         if (i == param->lmd->lmd_lmm.lmm_stripe_count) {

-                                if (param->exclude_obd)
-                                        goto obd_matches;
-                                goto decided;
+                                if (!param->exclude_obd)
+                                        goto decided;

                         }
                 }
         }
                         }
                 }
         }


@@ -2149,22 +2153,32 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
         }
 
         /* Check the time on mds. */
         }
 
         /* Check the time on mds. */

-        if (!decision) {
+        decision = 1;
+        if (param->atime || param->ctime || param->mtime) {

                 int for_mds;
 
                 for_mds = lustre_fs ? (S_ISREG(st->st_mode) &&
                                        param->lmd->lmd_lmm.lmm_stripe_count)
                                     : 0;
                 decision = find_time_check(st, param, for_mds);
                 int for_mds;
 
                 for_mds = lustre_fs ? (S_ISREG(st->st_mode) &&
                                        param->lmd->lmd_lmm.lmm_stripe_count)
                                     : 0;
                 decision = find_time_check(st, param, for_mds);

+                if (decision == -1)
+                        goto decided;

         }
 
         }
 

-obd_matches:

         /* If file still fits the request, ask ost for updated info.
            The regular stat is almost of the same speed as some new
            'glimpse-size-ioctl'. */
         /* If file still fits the request, ask ost for updated info.
            The regular stat is almost of the same speed as some new
            'glimpse-size-ioctl'. */

-        if (!decision && S_ISREG(st->st_mode) &&
-            param->lmd->lmd_lmm.lmm_stripe_count &&
-            (param->check_size ||param->atime || param->mtime || param->ctime)) {
+
+        if (param->check_size && S_ISREG(st->st_mode) &&
+            param->lmd->lmd_lmm.lmm_stripe_count)
+                decision = 0;
+
+        while (!decision) {
+                /* For regular files with the stripe the decision may have not
+                 * been taken yet if *time or size is to be checked. */
+                LASSERT(S_ISREG(st->st_mode) &&
+                        param->lmd->lmd_lmm.lmm_stripe_count);
+

                 if (param->obdindex != OBD_NOT_FOUND) {
                         /* Check whether the obd is active or not, if it is
                          * not active, just print the object affected by this
                 if (param->obdindex != OBD_NOT_FOUND) {
                         /* Check whether the obd is active or not, if it is
                          * not active, just print the object affected by this


@@ -2183,7 +2197,7 @@ obd_matches:
                                              "obd_uuid: %s failed %s ",
                                              param->obduuid->uuid,
                                              strerror(errno));
                                              "obd_uuid: %s failed %s ",
                                              param->obduuid->uuid,
                                              strerror(errno));

-                                goto print_path;
+                                break;

                         }
                 }
                 if (dir) {
                         }
                 }
                 if (dir) {


@@ -2213,6 +2227,8 @@ obd_matches:
                 decision = find_time_check(st, param, 0);
                 if (decision == -1)
                         goto decided;
                 decision = find_time_check(st, param, 0);
                 if (decision == -1)
                         goto decided;

+
+                break;

         }
 
         if (param->check_size)
         }
 
         if (param->check_size)


@@ -2220,7 +2236,6 @@ obd_matches:
                                           param->size_sign, param->exclude_size,
                                           param->size_units, 0);
 
                                           param->size_sign, param->exclude_size,
                                           param->size_units, 0);
 

-print_path:

         if (decision != -1) {
                 llapi_printf(LLAPI_MSG_NORMAL, "%s", path);
                 if (param->zeroend)
         if (decision != -1) {
                 llapi_printf(LLAPI_MSG_NORMAL, "%s", path);
                 if (param->zeroend)