From 34e647b06a6ff3835804d8a3796959420156fc4f Mon Sep 17 00:00:00 2001 From: Vitaly Fertman Date: Thu, 26 May 2011 01:07:09 +0400 Subject: [PATCH] LU-575 MRP-133 lfs find speedup lfs find should send getattr on mds only if needed; lfs find should not break on matched obd but check other parameters as well; lfs find time compare fixes; Reviewed-by: Alexey Lyashkov Reviewed-by: Colin Faber Signed-off-by: Vitaly Fertman LU-575 MRP-260 fix quota tests a fix for quota size units which conflicted with lfind size units Reviewed-by: Alexey Lyashkov Reviewed-by: Andrew Perepechko Signed-off-by: Vitaly Fertman Signed-off-by: Bob Glossman Change-Id: Id0484955602d6504b622006207ba7be4f183529f Reviewed-on: http://review.whamcloud.com/2644 Tested-by: Hudson Reviewed-by: Andreas Dilger Tested-by: Maloo --- lustre/tests/sanity.sh | 25 ++++++++++---- lustre/utils/lfs.c | 10 +++--- lustre/utils/liblustreapi.c | 79 +++++++++++++++++++++++++++------------------ 3 files changed, 71 insertions(+), 43 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index c88c864..7ce344b 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -3402,19 +3402,27 @@ test_56n() { run_test 56n "check lfs find -type l =============================" test_56o() { - setup_56 $NUMFILES $NUMDIRS TDIR=$DIR/${tdir}g + rm -rf $TDIR + setup_56 $NUMFILES $NUMDIRS utime $TDIR/file1 > /dev/null || error "utime (1)" utime $TDIR/file2 > /dev/null || error "utime (2)" utime $TDIR/dir1 > /dev/null || error "utime (3)" utime $TDIR/dir2 > /dev/null || error "utime (4)" utime $TDIR/dir1/file1 > /dev/null || error "utime (5)" + dd if=/dev/zero count=1 >> $TDIR/dir1/file1 && sync + + EXPECTED=4 + NUMS=`$LFIND -mtime +0 $TDIR | wc -l` + [ $NUMS -eq $EXPECTED ] || \ + error "lfs find -mtime +0 $TDIR wrong: found $NUMS, expected $EXPECTED" - EXPECTED=5 - NUMS=`$LFIND -mtime +1 $TDIR | wc -l` + EXPECTED=12 + NUMS=`$LFIND -mtime 0 $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ - error "lfs find -mtime $TDIR wrong: found $NUMS, expected $EXPECTED" + error "lfs find -mtime 0 $TDIR wrong: found $NUMS, expected $EXPECTED" + } run_test 56o "check lfs find -mtime for old files ==========================" @@ -3477,6 +3485,7 @@ test_56r() { [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR ! -size 0 wrong: found $NUMS, expected $EXPECTED" echo "test" > $TDIR/56r && sync + echo "test2" > $TDIR/56r2 && sync EXPECTED=1 NUMS=`$LFIND -size 5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ @@ -3485,14 +3494,18 @@ test_56r() { NUMS=`$LFIND -size +5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR -size +5 wrong: found $NUMS, expected $EXPECTED" - EXPECTED=13 + EXPECTED=2 NUMS=`$LFIND -size +0 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR -size +0 wrong: found $NUMS, expected $EXPECTED" - EXPECTED=0 + EXPECTED=2 NUMS=`$LFIND ! -size -5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR ! -size -5 wrong: found $NUMS, expected $EXPECTED" + EXPECTED=12 + NUMS=`$LFIND -size -5 -t f $TDIR | wc -l` + [ $NUMS -eq $EXPECTED ] || \ + error "lfs find $TDIR -size -5 wrong: found $NUMS, expected $EXPECTED" } run_test 56r "check lfs find -size works ==========================" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 714f6b6..683556d 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -281,7 +281,7 @@ static int lfs_setstripe(int argc, char **argv) char *stripe_off_arg = NULL; char *stripe_count_arg = NULL; char *pool_name_arg = NULL; - unsigned long long size_units; + unsigned long long size_units = 1; struct option long_opts[] = { {"count", required_argument, 0, 'c'}, @@ -498,7 +498,7 @@ static int lfs_find(int argc, char **argv) int new_fashion = 1; int c, ret; time_t t; - struct find_param param = { .maxdepth = -1 }; + struct find_param param = { .maxdepth = -1, .size_units = 0 }; struct option long_opts[] = { {"atime", required_argument, 0, 'A'}, {"ctime", required_argument, 0, 'C'}, @@ -1609,9 +1609,9 @@ error: return ULONG_MAX; } -#define ARG2ULL(nr, str, defscale) \ +#define ARG2ULL(nr, str, def_units) \ do { \ - unsigned long long limit, units = 0; \ + unsigned long long limit, units = def_units; \ int rc; \ \ rc = parse_size(str, &limit, &units, 1); \ @@ -1619,7 +1619,7 @@ do { \ fprintf(stderr, "error: bad limit value %s\n", str); \ return CMD_HELP; \ } \ - nr = ((units == 0) ? (defscale) : 1) * limit; \ + nr = limit; \ } while (0) static inline int has_times_option(int argc, char **argv) diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 3d40ea7..9fcb219 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -170,13 +170,16 @@ void llapi_printf(int level, char *fmt, ...) } /** - * size_units is unchanged if no specifier used + * size_units is to be initialized (or zeroed) by caller. */ int parse_size(char *optarg, unsigned long long *size, unsigned long long *size_units, int bytes_spec) { char *end; + if (*size_units == 0) + *size_units = 1; + if (strncmp(optarg, "-", 1) == 0) return -1; @@ -186,7 +189,6 @@ int parse_size(char *optarg, unsigned long long *size, if ((*end == 'b') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 9))) == 0 && !bytes_spec) { - *size <<= 9; *size_units = 1 << 9; } else if ((*end == 'b') && *(end+1) == '\0' && bytes_spec) { @@ -194,38 +196,32 @@ int parse_size(char *optarg, unsigned long long *size, } else if ((*end == 'k' || *end == 'K') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 10))) == 0) { - *size <<= 10; *size_units = 1 << 10; } else if ((*end == 'm' || *end == 'M') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 20))) == 0) { - *size <<= 20; *size_units = 1 << 20; } else if ((*end == 'g' || *end == 'G') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 30))) == 0) { - *size <<= 30; *size_units = 1 << 30; } else if ((*end == 't' || *end == 'T') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 40))) == 0) { - *size <<= 40; *size_units = 1ULL << 40; } else if ((*end == 'p' || *end == 'P') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 50))) == 0) { - *size <<= 50; *size_units = 1ULL << 50; } else if ((*end == 'e' || *end == 'E') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 60))) == 0) { - *size <<= 60; *size_units = 1ULL << 60; } else { return -1; } } - + *size *= *size_units; return 0; } @@ -1882,7 +1878,7 @@ int llapi_file_lookup(int dirfd, const char *name) * sign), 1st column is the answer for the MDS value, the 2nd is for the OST: * -------------------------------------- * 1 | file > limit; sign > 0 | -1 / -1 | - * 2 | file = limit; sign > 0 | ? / 1 | + * 2 | file = limit; sign > 0 | -1 / -1 | * 3 | file < limit; sign > 0 | ? / 1 | * 4 | file > limit; sign = 0 | -1 / -1 | * 5 | file = limit; sign = 0 | ? / 1 | <- (see the Note below) @@ -1899,15 +1895,16 @@ static int find_value_cmp(unsigned int file, unsigned int limit, int sign, int ret = -1; if (sign > 0) { - if (file <= limit) + /* Drop the fraction of margin (of days). */ + if (file + margin <= limit) ret = mds ? 0 : 1; } else if (sign == 0) { - if (file <= limit && file + margin >= limit) + if (file <= limit && file + margin > limit) ret = mds ? 0 : 1; else if (file + margin <= limit) ret = mds ? 0 : -1; } else if (sign < 0) { - if (file >= limit) + if (file > limit) ret = 1; else if (mds) ret = 0; @@ -1926,7 +1923,7 @@ static int find_value_cmp(unsigned int file, unsigned int limit, int sign, static int find_time_check(lstat_t *st, struct find_param *param, int mds) { int ret; - int rc = 0; + int rc = 1; /* Check if file is accepted. */ if (param->atime) { @@ -1979,7 +1976,8 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, LASSERT(parent != NULL || dir != NULL); - param->lmd->lmd_lmm.lmm_stripe_count = 0; + if (param->have_fileinfo == 0) + param->lmd->lmd_lmm.lmm_stripe_count = 0; /* If a regular expression is presented, make the initial decision */ if (param->pattern != NULL) { @@ -2005,14 +2003,18 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, } - /* If a time or OST should be checked, the decision is not taken yet. */ - if (param->atime || param->ctime || param->mtime || param->obduuid || - param->check_size) + ret = 0; + + /* Request MDS for the stat info if some of these parameters need + * to be compared. */ + if (param->obduuid || param->check_uid || param->check_gid || + param->check_pool || param->atime || param->ctime || + param->mtime || param->check_size) + decision = 0; + if (param->type && checked_type == 0) decision = 0; - ret = 0; - /* Request MDS for the stat info. */ - if (param->have_fileinfo == 0) { + if (param->have_fileinfo == 0 && decision == 0) { if (dir) { /* retrieve needed file info */ ret = ioctl(dirfd(dir), LL_IOC_MDC_GETINFO, @@ -2123,15 +2125,17 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, lmm_objects[i].l_ost_idx) { if (param->exclude_obd) goto decided; - goto obd_matches; + break; } } + /* If an OBD matches, just break */ + if (j != param->num_obds) + break; } if (i == param->lmd->lmd_lmm.lmm_stripe_count) { - if (param->exclude_obd) - goto obd_matches; - goto decided; + if (!param->exclude_obd) + goto decided; } } } @@ -2176,22 +2180,32 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, } /* Check the time on mds. */ - if (!decision) { + decision = 1; + if (param->atime || param->ctime || param->mtime) { int for_mds; for_mds = lustre_fs ? (S_ISREG(st->st_mode) && param->lmd->lmd_lmm.lmm_stripe_count) : 0; decision = find_time_check(st, param, for_mds); + if (decision == -1) + goto decided; } -obd_matches: /* If file still fits the request, ask ost for updated info. The regular stat is almost of the same speed as some new 'glimpse-size-ioctl'. */ - if (!decision && S_ISREG(st->st_mode) && - param->lmd->lmd_lmm.lmm_stripe_count && - (param->check_size ||param->atime || param->mtime || param->ctime)) { + + if (param->check_size && S_ISREG(st->st_mode) && + param->lmd->lmd_lmm.lmm_stripe_count) + decision = 0; + + while (!decision) { + /* For regular files with the stripe the decision may have not + * been taken yet if *time or size is to be checked. */ + LASSERT(S_ISREG(st->st_mode) && + param->lmd->lmd_lmm.lmm_stripe_count); + if (param->obdindex != OBD_NOT_FOUND) { /* Check whether the obd is active or not, if it is * not active, just print the object affected by this @@ -2210,7 +2224,7 @@ obd_matches: "obd_uuid: %s failed %s ", param->obduuid->uuid, strerror(errno)); - goto print_path; + break; } } if (dir) { @@ -2240,6 +2254,8 @@ obd_matches: decision = find_time_check(st, param, 0); if (decision == -1) goto decided; + + break; } if (param->check_size) @@ -2247,7 +2263,6 @@ obd_matches: param->size_sign, param->exclude_size, param->size_units, 0); -print_path: if (decision != -1) { llapi_printf(LLAPI_MSG_NORMAL, "%s", path); if (param->zeroend) -- 1.8.3.1