From 8583479b1249fa154cb0944f9a3aed43c2e549ce Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Wed, 20 Nov 2019 23:12:10 +0800 Subject: [PATCH] LU-12984 utils: Add -newerXY support for lfs find In regular "find" it is possible to specify a filename to use for a relative timestamp: -newerXY reference Compares the timestamp of the current file with reference. The reference argument is normally the name of a file (and one of its timestamps is used for the comparison) but it may also be a string describing an absolute time. X and Y are placeholders for other letters, and these letters select which time belonging to how reference is used for the comparison. - a The access time of the file reference - c The inode status change time of reference - m The modification time of the file reference - t reference is interpreted directly as a time We should enhance Lustre 'lfs find' to support '-newerXY' options. In current implementation, When reference is interpreted directly as a time, it must be in one of the following formats: - "%Y-%m-%d %H:%M:%S" - "%Y-%m-%d %H:%M" - "%Y-%m-%d" - "%H:%M:%S" - "%H:%M" - "@%s" - "%s" Otherwise, it will report errors. Signed-off-by: Qian Yingjin Change-Id: Ib006940b231c4a18f892c492dd892f7733b5d8ba Reviewed-on: https://review.whamcloud.com/36806 Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: Li Xi Tested-by: Maloo --- lustre/doc/lfs-find.1 | 38 ++++++++++ lustre/include/lustre/lustreapi.h | 15 +++- lustre/tests/sanity.sh | 59 +++++++++++++++ lustre/utils/lfs.c | 155 +++++++++++++++++++++++++++++++++++++- lustre/utils/liblustreapi.c | 72 +++++++++++++++++- 5 files changed, 335 insertions(+), 4 deletions(-) diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 041869d..9181355 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -25,6 +25,7 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--mirror-state\fR <[^]\fIstate\fR>] [[\fB!\fR] \fB--mtime\fR|\fB-M\fR [\fB-+\fR]\fIn[smhdwy]\fR] [[\fB!\fR] \fB--name\fR|\fB-n <\fIpattern\fR>] + [[\fB!\fR] \fB--newer\fR[\fBXY\fR] <\fIreference\fR>] [[\fB!\fR] \fB--ost\fR|\fB-O\fR <\fIindex\fR,...>] [[\fB!\fR] \fB--pool\fR <\fIpool\fR>] [\fB--print\fR|\fB-P\fR] @@ -195,6 +196,43 @@ standard .BR glob (7) file name regular expressions and wildcards. .TP +.BR -newer [ XY "] " \fIreference +Succeeds if timestamp \fIX\fR of the file being considered is newer +than timestamp \fIY\fR of the file +.IR reference . +The letters \fIX\fR and \fIY\fR can be any of the following letters: + +.TS +ll +ll +ll +ll +llw(2i). +a The access time of the file \fIreference\fR +c The inode status change time of \fIreference\fR +m The modification time of the file \fIreference\fR +t \fIreference\fR is interpreted directly as a time +.TE + +Some combinations are invalid; for example, it is invalid for +.I X +to be +.IR t . +Specifying +.B -newer +is equivalent to +.BR -newermm . +When +.IR reference +is interpreted directly as a time, currently it must be in one of the +following formats: "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", +"%H:%M:%S", "%H:%M", to represent year, month, day, hour, minute, seconds, +with unspecified times at the start of that minute or day, unspecified dates +being "today", and "@%s" or "%s" the seconds since the Unix epoch (see +.BR strftime (3) +for details of the time formats). Otherwise, it will report an error. + +.TP .BR --ost | -O File has an object on the specified OST(s). The OST names can be specified using the whole OST target name, or just the OST index number. If multiple diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index bd1f940..5efe265 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -188,6 +188,13 @@ enum llapi_layout_verbose { #define VERBOSE_OFFSET VERBOSE_STRIPE_OFFSET #define VERBOSE_LAYOUT VERBOSE_PATTERN +enum { + NEWERXY_ATIME = 0, /* neweraY */ + NEWERXY_MTIME = 1, /* newermY */ + NEWERXY_CTIME = 2, /* newercY */ + NEWERXY_MAX, +}; + struct find_param { unsigned int fp_max_depth; dev_t fp_dev; @@ -279,7 +286,7 @@ struct find_param { fp_check_ext_size:1, /* extension size */ fp_exclude_ext_size:1, fp_lazy:1, - fp_unused_bit1:1, + fp_newerxy:1, fp_unused_bit2:1, /* All of these unused bit */ fp_unused_bit3:1, /* fields available to use.*/ fp_unused_bit4:1, /* Once all unused fields */ @@ -345,6 +352,12 @@ struct find_param { __u32 fp_foreign_type; unsigned long long fp_ext_size; unsigned long long fp_ext_size_units; + + /* + * fp_newery[NEWERXY_MAX][0]: --newerXY reference + * fp_newery[NEWERXY_MAX][1]: ! -- newerXY reference + */ + time_t fp_newery[NEWERXY_MAX][2]; }; int llapi_ostlist(char *path, struct find_param *param); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 328a186..b62e7fd 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -5744,6 +5744,65 @@ test_56ob() { } run_test 56ob "check lfs find -atime -mtime -ctime with units" +test_newerXY_base() { + local x=$1 + local y=$2 + local dir=$DIR/$tdir + local ref + local negref + + if [ $y == "t" ]; then + ref="\"$(date +"%Y-%m-%d %H:%M:%S")\"" + else + ref=$DIR/$tfile.newer + touch $ref || error "touch $ref failed" + fi + sleep 2 + setup_56 $dir $NUMFILES $NUMDIRS "-i0 -c1" "-i0 -c1" + sleep 2 + if [ $y == "t" ]; then + negref="\"$(date +"%Y-%m-%d %H:%M:%S")\"" + else + negref=$DIR/$tfile.newerneg + touch $negref || error "touch $negref failed" + fi + + local cmd="$LFS find $dir -newer$x$y $ref" + local nums=$(eval $cmd | wc -l) + local expected=$(((NUMFILES + 2) * NUMDIRS + 1)) + + [ $nums -eq $expected ] || + error "'$cmd' wrong: found $nums, expected $expected" + + cmd="$LFS find $dir ! -newer$x$y $negref" + nums=$(eval $cmd | wc -l) + [ $nums -eq $expected ] || + error "'$cmd' wrong: found $nums, expected $expected" + + cmd="$LFS find $dir -newer$x$y $ref ! -newer$x$y $negref" + nums=$(eval $cmd | wc -l) + [ $nums -eq $expected ] || + error "'$cmd' wrong: found $nums, expected $expected" + + rm -rf $DIR/* +} + +test_56oc() { + test_newerXY_base "a" "a" + test_newerXY_base "a" "m" + test_newerXY_base "a" "c" + test_newerXY_base "m" "a" + test_newerXY_base "m" "m" + test_newerXY_base "m" "c" + test_newerXY_base "c" "a" + test_newerXY_base "c" "m" + test_newerXY_base "c" "c" + test_newerXY_base "a" "t" + test_newerXY_base "m" "t" + test_newerXY_base "c" "t" +} +run_test 56oc "check lfs find -newerXY work" + test_56p() { [ $RUNAS_ID -eq $UID ] && skip_env "RUNAS_ID = UID = $UID -- skipping" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 6160ea4..dc1f0e6 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -457,6 +457,7 @@ command_t cmdlist[] = { "usage: find ...\n" " [[!] --atime|-A [+-]N[smhdwy]] [[!] --ctime|-C [+-]N[smhdwy]]\n" " [[!] --mtime|-M [+-]N[smhdwy]] [[!] --blocks|-b N]\n" + " [[!] --newer[XY] ]\n" " [--maxdepth|-D N] [[!] --mdt-index|--mdt|-m ]\n" " [[!] --name|-n ] [[!] --ost|-O ]\n" " [--print|-P] [--print0|-0] [[!] --size|-s [+-]N[bkMGTPE]]\n" @@ -2976,6 +2977,7 @@ enum { LFS_MIRROR_INDEX_OPT, LFS_LAYOUT_FOREIGN_OPT, LFS_MODE_OPT, + LFS_NEWERXY_OPT, }; /* functions */ @@ -4125,6 +4127,32 @@ static int lfs_find(int argc, char **argv) .name = "mirror-state", .has_arg = required_argument }, { .val = LFS_LAYOUT_FOREIGN_OPT, .name = "foreign", .has_arg = optional_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newer", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "neweraa", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "neweram", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newerac", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newerma", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newermm", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newermc", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newerca", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newercm", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newercc", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newerat", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newermt", .has_arg = required_argument}, + { .val = LFS_NEWERXY_OPT, + .name = "newerct", .has_arg = required_argument}, { .val = 'c', .name = "stripe-count", .has_arg = required_argument }, { .val = 'c', .name = "stripe_count", .has_arg = required_argument }, { .val = 'C', .name = "ctime", .has_arg = required_argument }, @@ -4176,6 +4204,7 @@ static int lfs_find(int argc, char **argv) /* getstripe { .val = 'v', .name = "verbose", .has_arg = no_argument }, */ /* getstripe { .val = 'y', .name = "yaml", .has_arg = no_argument }, */ { .name = NULL } }; + int optidx = 0; int pathstart = -1; int pathend = -1; int pathbad = -1; @@ -4190,7 +4219,7 @@ static int lfs_find(int argc, char **argv) /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */ while ((c = getopt_long_only(argc, argv, "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:vz:", - long_opts, NULL)) >= 0) { + long_opts, &optidx)) >= 0) { xtime = NULL; xsign = NULL; if (neg_opt) @@ -4412,6 +4441,130 @@ static int lfs_find(int argc, char **argv) param.fp_exclude_foreign = !!neg_opt; break; } + case LFS_NEWERXY_OPT: { + char x = 'm'; + char y = 'm'; + int xidx; + int negidx; + time_t *newery; + time_t ref = time(NULL); + + /* no need to check bad options, they won't get here */ + if (strlen(long_opts[optidx].name) == 7) { + x = long_opts[optidx].name[5]; + y = long_opts[optidx].name[6]; + } + + if (y == 't') { + static const char *const fmts[] = { + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y-%m-%d", + "%H:%M:%S", /* sometime today */ + "%H:%M", + "@%s", + "%s", + NULL }; + struct tm tm; + bool found = false; + int i; + + for (i = 0; fmts[i] != NULL; i++) { + char *ptr; + + /* Init for times relative to today */ + if (strncmp(fmts[i], "%H", 2) == 0) + localtime_r(&ref, &tm); + else + memset(&tm, 0, sizeof(tm)); + ptr = strptime(optarg, fmts[i], &tm); + /* Skip spaces */ + while (ptr && isspace(*ptr)) + ptr++; + if (ptr == optarg + strlen(optarg)) { + found = true; + break; + } + } + + if (!found) { + fprintf(stderr, + "%s: invalid time '%s'\n", + progname, optarg); + fprintf(stderr, + "supported formats are:\n "); + for (i = 0; fmts[i] != NULL; i++) + fprintf(stderr, "'%s', ", + fmts[i]); + fprintf(stderr, "\n"); + ret = -EINVAL; + goto err; + } + + ref = mktime(&tm); + } else { + struct stat statbuf; + + if (stat(optarg, &statbuf) < 0) { + fprintf(stderr, + "%s: cannot stat file '%s': %s\n", + progname, optarg, + strerror(errno)); + ret = -errno; + goto err; + } + + switch (y) { + case 'a': + ref = statbuf.st_atime; + break; + case 'm': + ref = statbuf.st_mtime; + break; + case 'c': + ref = statbuf.st_ctime; + break; + default: + fprintf(stderr, + "%s: invalid Y argument: '%c'\n", + progname, x); + ret = -EINVAL; + goto err; + } + } + + switch (x) { + case 'a': + xidx = NEWERXY_ATIME; + break; + case 'm': + xidx = NEWERXY_MTIME; + break; + case 'c': + xidx = NEWERXY_CTIME; + break; + default: + fprintf(stderr, + "%s: invalid X argument: '%c'\n", + progname, x); + ret = -EINVAL; + goto err; + } + + negidx = !!neg_opt; + newery = ¶m.fp_newery[xidx][negidx]; + + if (*newery == 0) { + *newery = ref; + } else { + if (negidx) + *newery = *newery > ref ? ref : *newery; + else + *newery = *newery > ref ? *newery : ref; + } + param.fp_newerxy = 1; + break; + } case 'g': case 'G': rc = name2gid(¶m.fp_gid, optarg); diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 77998f4..4238208 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -4044,6 +4044,57 @@ static int find_time_check(lstatx_t *stx, struct find_param *param, int mds) return rc; } +static int find_newerxy_check(lstatx_t *stx, struct find_param *param, int mds) +{ + int i; + int rc = 1; + int rc2; + + for (i = 0; i < 2; i++) { + /* Check if file is accepted. */ + if (param->fp_newery[NEWERXY_ATIME][i]) { + rc2 = find_value_cmp(stx->stx_atime.tv_sec, + param->fp_newery[NEWERXY_ATIME][i], + -1, i, 0, mds); + if (rc2 < 0) + return rc2; + rc = rc2; + } + + if (param->fp_newery[NEWERXY_MTIME][i]) { + rc2 = find_value_cmp(stx->stx_mtime.tv_sec, + param->fp_newery[NEWERXY_MTIME][i], + -1, i, 0, mds); + if (rc2 < 0) + return rc2; + + /* + * If the previous check matches, but this one is not + * yet clear, we should return 0 to do an RPC on OSTs. + */ + if (rc == 1) + rc = rc2; + } + + if (param->fp_newery[NEWERXY_CTIME][i]) { + rc2 = find_value_cmp(stx->stx_ctime.tv_sec, + param->fp_newery[NEWERXY_CTIME][i], + -1, i, 0, mds); + if (rc2 < 0) + return rc2; + + /* + * If the previous check matches, but this one is not + * yet clear, we should return 0 to do an RPC on OSTs. + */ + if (rc == 1) + rc = rc2; + } + } + + return rc; +} + /** * Check whether the stripes matches the indexes user provided * 1 : matched @@ -4555,7 +4606,7 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, /* Request MDS for the stat info if some of these parameters need * to be compared. */ if (param->fp_obd_uuid || param->fp_mdt_uuid || - param->fp_check_uid || param->fp_check_gid || + param->fp_check_uid || param->fp_check_gid || param->fp_newerxy || param->fp_atime || param->fp_mtime || param->fp_ctime || param->fp_check_size || param->fp_check_blocks || find_check_lmm_info(param) || @@ -4835,12 +4886,22 @@ obd_matches: int for_mds; for_mds = lustre_fs ? - (S_ISREG(stx->stx_mode) && stripe_count) : 0; + (S_ISREG(stx->stx_mode) && stripe_count) : 0; decision = find_time_check(stx, param, for_mds); if (decision == -1) goto decided; } + if (param->fp_newerxy) { + int for_mds; + + for_mds = lustre_fs ? + (S_ISREG(stx->stx_mode) && stripe_count) : 0; + decision = find_newerxy_check(stx, param, for_mds); + if (decision == -1) + goto decided; + } + flags = param->fp_lmd->lmd_flags; if (param->fp_check_size && ((S_ISREG(stx->stx_mode) && stripe_count) || @@ -4899,6 +4960,13 @@ obd_matches: decision = find_time_check(stx, param, 0); if (decision == -1) goto decided; + + if (param->fp_newerxy) { + decision = find_newerxy_check(stx, param, 0); + if (decision == -1) + goto decided; + + } } if (param->fp_check_size) { -- 1.8.3.1