From 8cb2e445339effe9bb3db23df47595a043b7e4ff Mon Sep 17 00:00:00 2001 From: Vitaly Fertman Date: Thu, 26 May 2011 01:07:09 +0400 Subject: [PATCH] LU-575 MRP-133 lfs find speedup lfs find should send getattr on mds only if needed; lfs find should not break on matched obd but check other parameters as well; lfs find time compare fixes; Reviewed-by: Alexey Lyashkov Reviewed-by: Colin Faber Signed-off-by: Vitaly Fertman LU-575 MRP-260 fix quota tests a fix for quota size units which conflicted with lfind size units Reviewed-by: Alexey Lyashkov Reviewed-by: Andrew Perepechko Signed-off-by: Vitaly Fertman Change-Id: I42b2acee18c6c9e0d787b97a5e957431d2bb4bf1 Reviewed-on: http://review.whamcloud.com/1195 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- lustre/tests/sanity.sh | 25 ++++++++++---- lustre/utils/lfs.c | 10 +++--- lustre/utils/liblustreapi.c | 79 +++++++++++++++++++++++++++------------------ 3 files changed, 71 insertions(+), 43 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index c50b040..9aa27b2 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -3408,19 +3408,27 @@ test_56n() { run_test 56n "check lfs find -type l =============================" test_56o() { - setup_56 $NUMFILES $NUMDIRS TDIR=$DIR/${tdir}g + rm -rf $TDIR + setup_56 $NUMFILES $NUMDIRS utime $TDIR/file1 > /dev/null || error "utime (1)" utime $TDIR/file2 > /dev/null || error "utime (2)" utime $TDIR/dir1 > /dev/null || error "utime (3)" utime $TDIR/dir2 > /dev/null || error "utime (4)" utime $TDIR/dir1/file1 > /dev/null || error "utime (5)" + dd if=/dev/zero count=1 >> $TDIR/dir1/file1 && sync + + EXPECTED=4 + NUMS=`$LFIND -mtime +0 $TDIR | wc -l` + [ $NUMS -eq $EXPECTED ] || \ + error "lfs find -mtime +0 $TDIR wrong: found $NUMS, expected $EXPECTED" - EXPECTED=5 - NUMS=`$LFIND -mtime +1 $TDIR | wc -l` + EXPECTED=12 + NUMS=`$LFIND -mtime 0 $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ - error "lfs find -mtime $TDIR wrong: found $NUMS, expected $EXPECTED" + error "lfs find -mtime 0 $TDIR wrong: found $NUMS, expected $EXPECTED" + } run_test 56o "check lfs find -mtime for old files ==========================" @@ -3483,6 +3491,7 @@ test_56r() { [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR ! -size 0 wrong: found $NUMS, expected $EXPECTED" echo "test" > $TDIR/56r && sync + echo "test2" > $TDIR/56r2 && sync EXPECTED=1 NUMS=`$LFIND -size 5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ @@ -3491,14 +3500,18 @@ test_56r() { NUMS=`$LFIND -size +5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR -size +5 wrong: found $NUMS, expected $EXPECTED" - EXPECTED=13 + EXPECTED=2 NUMS=`$LFIND -size +0 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR -size +0 wrong: found $NUMS, expected $EXPECTED" - EXPECTED=0 + EXPECTED=2 NUMS=`$LFIND ! -size -5 -t f $TDIR | wc -l` [ $NUMS -eq $EXPECTED ] || \ error "lfs find $TDIR ! -size -5 wrong: found $NUMS, expected $EXPECTED" + EXPECTED=12 + NUMS=`$LFIND -size -5 -t f $TDIR | wc -l` + [ $NUMS -eq $EXPECTED ] || \ + error "lfs find $TDIR -size -5 wrong: found $NUMS, expected $EXPECTED" } run_test 56r "check lfs find -size works ==========================" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 6010105..99707f2 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -277,7 +277,7 @@ static int lfs_setstripe(int argc, char **argv) char *stripe_off_arg = NULL; char *stripe_count_arg = NULL; char *pool_name_arg = NULL; - unsigned long long size_units; + unsigned long long size_units = 1; struct option long_opts[] = { {"count", required_argument, 0, 'c'}, @@ -494,7 +494,7 @@ static int lfs_find(int argc, char **argv) int new_fashion = 1; int c, ret; time_t t; - struct find_param param = { .maxdepth = -1 }; + struct find_param param = { .maxdepth = -1, .size_units = 0 }; struct option long_opts[] = { {"atime", required_argument, 0, 'A'}, {"ctime", required_argument, 0, 'C'}, @@ -1565,9 +1565,9 @@ error: return ULONG_MAX; } -#define ARG2ULL(nr, str, defscale) \ +#define ARG2ULL(nr, str, def_units) \ do { \ - unsigned long long limit, units = 0; \ + unsigned long long limit, units = def_units; \ int rc; \ \ rc = parse_size(str, &limit, &units, 1); \ @@ -1575,7 +1575,7 @@ do { \ fprintf(stderr, "error: bad limit value %s\n", str); \ return CMD_HELP; \ } \ - nr = ((units == 0) ? (defscale) : 1) * limit; \ + nr = limit; \ } while (0) static inline int has_times_option(int argc, char **argv) diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index f935ded..e08bef0 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -170,20 +170,22 @@ void llapi_printf(int level, char *fmt, ...) } /** - * size_units is unchanged if no specifier used + * size_units is to be initialized (or zeroed) by caller. */ int parse_size(char *optarg, unsigned long long *size, unsigned long long *size_units, int bytes_spec) { char *end; + if (*size_units == 0) + *size_units = 1; + *size = strtoull(optarg, &end, 0); if (*end != '\0') { if ((*end == 'b') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 9))) == 0 && !bytes_spec) { - *size <<= 9; *size_units = 1 << 9; } else if ((*end == 'b') && *(end+1) == '\0' && bytes_spec) { @@ -191,38 +193,32 @@ int parse_size(char *optarg, unsigned long long *size, } else if ((*end == 'k' || *end == 'K') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 10))) == 0) { - *size <<= 10; *size_units = 1 << 10; } else if ((*end == 'm' || *end == 'M') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 20))) == 0) { - *size <<= 20; *size_units = 1 << 20; } else if ((*end == 'g' || *end == 'G') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 30))) == 0) { - *size <<= 30; *size_units = 1 << 30; } else if ((*end == 't' || *end == 'T') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 40))) == 0) { - *size <<= 40; *size_units = 1ULL << 40; } else if ((*end == 'p' || *end == 'P') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 50))) == 0) { - *size <<= 50; *size_units = 1ULL << 50; } else if ((*end == 'e' || *end == 'E') && *(end+1) == '\0' && (*size & (~0ULL << (64 - 60))) == 0) { - *size <<= 60; *size_units = 1ULL << 60; } else { return -1; } } - + *size *= *size_units; return 0; } @@ -1854,7 +1850,7 @@ int llapi_file_lookup(int dirfd, const char *name) * sign), 1st column is the answer for the MDS value, the 2nd is for the OST: * -------------------------------------- * 1 | file > limit; sign > 0 | -1 / -1 | - * 2 | file = limit; sign > 0 | ? / 1 | + * 2 | file = limit; sign > 0 | -1 / -1 | * 3 | file < limit; sign > 0 | ? / 1 | * 4 | file > limit; sign = 0 | -1 / -1 | * 5 | file = limit; sign = 0 | ? / 1 | <- (see the Note below) @@ -1872,15 +1868,16 @@ static int find_value_cmp(unsigned long long file, unsigned long long limit, int ret = -1; if (sign > 0) { - if (file <= limit) + /* Drop the fraction of margin (of days). */ + if (file + margin <= limit) ret = mds ? 0 : 1; } else if (sign == 0) { - if (file <= limit && file + margin >= limit) + if (file <= limit && file + margin > limit) ret = mds ? 0 : 1; else if (file + margin <= limit) ret = mds ? 0 : -1; } else if (sign < 0) { - if (file >= limit) + if (file > limit) ret = 1; else if (mds) ret = 0; @@ -1899,7 +1896,7 @@ static int find_value_cmp(unsigned long long file, unsigned long long limit, static int find_time_check(lstat_t *st, struct find_param *param, int mds) { int ret; - int rc = 0; + int rc = 1; /* Check if file is accepted. */ if (param->atime) { @@ -1952,7 +1949,8 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, LASSERT(parent != NULL || dir != NULL); - param->lmd->lmd_lmm.lmm_stripe_count = 0; + if (param->have_fileinfo == 0) + param->lmd->lmd_lmm.lmm_stripe_count = 0; /* If a regular expression is presented, make the initial decision */ if (param->pattern != NULL) { @@ -1978,14 +1976,18 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, } - /* If a time or OST should be checked, the decision is not taken yet. */ - if (param->atime || param->ctime || param->mtime || param->obduuid || - param->check_size) + ret = 0; + + /* Request MDS for the stat info if some of these parameters need + * to be compared. */ + if (param->obduuid || param->check_uid || param->check_gid || + param->check_pool || param->atime || param->ctime || + param->mtime || param->check_size) + decision = 0; + if (param->type && checked_type == 0) decision = 0; - ret = 0; - /* Request MDS for the stat info. */ - if (param->have_fileinfo == 0) { + if (param->have_fileinfo == 0 && decision == 0) { if (dir) { /* retrieve needed file info */ ret = ioctl(dirfd(dir), LL_IOC_MDC_GETINFO, @@ -2096,15 +2098,17 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, lmm_objects[i].l_ost_idx) { if (param->exclude_obd) goto decided; - goto obd_matches; + break; } } + /* If an OBD matches, just break */ + if (j != param->num_obds) + break; } if (i == param->lmd->lmd_lmm.lmm_stripe_count) { - if (param->exclude_obd) - goto obd_matches; - goto decided; + if (!param->exclude_obd) + goto decided; } } } @@ -2149,22 +2153,32 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir, } /* Check the time on mds. */ - if (!decision) { + decision = 1; + if (param->atime || param->ctime || param->mtime) { int for_mds; for_mds = lustre_fs ? (S_ISREG(st->st_mode) && param->lmd->lmd_lmm.lmm_stripe_count) : 0; decision = find_time_check(st, param, for_mds); + if (decision == -1) + goto decided; } -obd_matches: /* If file still fits the request, ask ost for updated info. The regular stat is almost of the same speed as some new 'glimpse-size-ioctl'. */ - if (!decision && S_ISREG(st->st_mode) && - param->lmd->lmd_lmm.lmm_stripe_count && - (param->check_size ||param->atime || param->mtime || param->ctime)) { + + if (param->check_size && S_ISREG(st->st_mode) && + param->lmd->lmd_lmm.lmm_stripe_count) + decision = 0; + + while (!decision) { + /* For regular files with the stripe the decision may have not + * been taken yet if *time or size is to be checked. */ + LASSERT(S_ISREG(st->st_mode) && + param->lmd->lmd_lmm.lmm_stripe_count); + if (param->obdindex != OBD_NOT_FOUND) { /* Check whether the obd is active or not, if it is * not active, just print the object affected by this @@ -2183,7 +2197,7 @@ obd_matches: "obd_uuid: %s failed %s ", param->obduuid->uuid, strerror(errno)); - goto print_path; + break; } } if (dir) { @@ -2213,6 +2227,8 @@ obd_matches: decision = find_time_check(st, param, 0); if (decision == -1) goto decided; + + break; } if (param->check_size) @@ -2220,7 +2236,6 @@ obd_matches: param->size_sign, param->exclude_size, param->size_units, 0); -print_path: if (decision != -1) { llapi_printf(LLAPI_MSG_NORMAL, "%s", path); if (param->zeroend) -- 1.8.3.1