Whamcloud - gitweb
LU-12984 utils: Add -newerXY support for lfs find 06/36806/9
authorQian Yingjin <qian@ddn.com>
Wed, 20 Nov 2019 15:12:10 +0000 (23:12 +0800)
committerOleg Drokin <green@whamcloud.com>
Mon, 16 Dec 2019 05:58:27 +0000 (05:58 +0000)
In regular "find" it is possible to specify a filename to use
for a relative timestamp:
-newerXY reference
 Compares the timestamp of the current file with reference.
 The reference argument is normally the name of a file (and
 one of its timestamps is used for the comparison) but it may
 also be a string describing an absolute time. X and Y are
 placeholders for other letters, and these letters select which
 time belonging to how reference is used for the comparison.
 - a   The access time of the file reference
 - c   The inode status change time of reference
 - m   The modification time of the file reference
 - t   reference is interpreted directly as a time

We should enhance Lustre 'lfs find' to support '-newerXY' options.
In current implementation, When reference is interpreted directly
as a time, it must be in one of the following formats:
- "%Y-%m-%d %H:%M:%S"
- "%Y-%m-%d %H:%M"
- "%Y-%m-%d"
- "%H:%M:%S"
- "%H:%M"
- "@%s"
- "%s"
Otherwise, it will report errors.

Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: Ib006940b231c4a18f892c492dd892f7733b5d8ba
Reviewed-on: https://review.whamcloud.com/36806
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Li Xi <lixi@ddn.com>
Tested-by: Maloo <maloo@whamcloud.com>
lustre/doc/lfs-find.1
lustre/include/lustre/lustreapi.h
lustre/tests/sanity.sh
lustre/utils/lfs.c
lustre/utils/liblustreapi.c

index 041869d..9181355 100644 (file)
@@ -25,6 +25,7 @@ lfs-find \- Lustre client utility to list files with specific attributes
 [[\fB!\fR] \fB--mirror-state\fR <[^]\fIstate\fR>]
       [[\fB!\fR] \fB--mtime\fR|\fB-M\fR [\fB-+\fR]\fIn[smhdwy]\fR]
 [[\fB!\fR] \fB--name\fR|\fB-n <\fIpattern\fR>]
+      [[\fB!\fR] \fB--newer\fR[\fBXY\fR] <\fIreference\fR>]
       [[\fB!\fR] \fB--ost\fR|\fB-O\fR <\fIindex\fR,...>]
 [[\fB!\fR] \fB--pool\fR <\fIpool\fR>]
 [\fB--print\fR|\fB-P\fR]
@@ -195,6 +196,43 @@ standard
 .BR glob (7)
 file name regular expressions and wildcards.
 .TP
+.BR -newer [ XY "] " \fIreference
+Succeeds if timestamp \fIX\fR of the file being considered is newer
+than timestamp \fIY\fR of the file
+.IR reference .
+The letters \fIX\fR and \fIY\fR can be any of the following letters:
+
+.TS
+ll
+ll
+ll
+ll
+llw(2i).
+a       The access time of the file \fIreference\fR
+c       The inode status change time of \fIreference\fR
+m       The modification time of the file \fIreference\fR
+t       \fIreference\fR is interpreted directly as a time
+.TE
+
+Some combinations are invalid; for example, it is invalid for
+.I X
+to be
+.IR t .
+Specifying
+.B -newer
+is equivalent to
+.BR -newermm .
+When
+.IR reference
+is interpreted directly as a time, currently it must be in one of the
+following formats: "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d",
+"%H:%M:%S", "%H:%M", to represent year, month, day, hour, minute, seconds,
+with unspecified times at the start of that minute or day, unspecified dates
+being "today", and "@%s" or "%s" the seconds since the Unix epoch (see
+.BR strftime (3)
+for details of the time formats).  Otherwise, it will report an error.
+
+.TP
 .BR --ost | -O
 File has an object on the specified OST(s).  The OST names can be specified
 using the whole OST target name, or just the OST index number. If multiple
index bd1f940..5efe265 100644 (file)
@@ -188,6 +188,13 @@ enum llapi_layout_verbose  {
 #define VERBOSE_OFFSET VERBOSE_STRIPE_OFFSET
 #define VERBOSE_LAYOUT VERBOSE_PATTERN
 
+enum {
+       NEWERXY_ATIME = 0,      /* neweraY */
+       NEWERXY_MTIME = 1,      /* newermY */
+       NEWERXY_CTIME = 2,      /* newercY */
+       NEWERXY_MAX,
+};
+
 struct find_param {
        unsigned int             fp_max_depth;
        dev_t                    fp_dev;
@@ -279,7 +286,7 @@ struct find_param {
                                 fp_check_ext_size:1, /* extension size */
                                 fp_exclude_ext_size:1,
                                 fp_lazy:1,
-                                fp_unused_bit1:1,
+                                fp_newerxy:1,
                                 fp_unused_bit2:1, /* All of these unused bit */
                                 fp_unused_bit3:1, /* fields available to use.*/
                                 fp_unused_bit4:1, /* Once all unused fields  */
@@ -345,6 +352,12 @@ struct find_param {
        __u32                    fp_foreign_type;
        unsigned long long       fp_ext_size;
        unsigned long long       fp_ext_size_units;
+
+       /*
+        * fp_newery[NEWERXY_MAX][0]: --newerXY reference
+        * fp_newery[NEWERXY_MAX][1]: ! -- newerXY reference
+        */
+       time_t                   fp_newery[NEWERXY_MAX][2];
 };
 
 int llapi_ostlist(char *path, struct find_param *param);
index 328a186..b62e7fd 100644 (file)
@@ -5744,6 +5744,65 @@ test_56ob() {
 }
 run_test 56ob "check lfs find -atime -mtime -ctime with units"
 
+test_newerXY_base() {
+       local x=$1
+       local y=$2
+       local dir=$DIR/$tdir
+       local ref
+       local negref
+
+       if [ $y == "t" ]; then
+               ref="\"$(date +"%Y-%m-%d %H:%M:%S")\""
+       else
+               ref=$DIR/$tfile.newer
+               touch $ref || error "touch $ref failed"
+       fi
+       sleep 2
+       setup_56 $dir $NUMFILES $NUMDIRS "-i0 -c1" "-i0 -c1"
+       sleep 2
+       if [ $y == "t" ]; then
+               negref="\"$(date +"%Y-%m-%d %H:%M:%S")\""
+       else
+               negref=$DIR/$tfile.newerneg
+               touch $negref || error "touch $negref failed"
+       fi
+
+       local cmd="$LFS find $dir -newer$x$y $ref"
+       local nums=$(eval $cmd | wc -l)
+       local expected=$(((NUMFILES + 2) * NUMDIRS + 1))
+
+       [ $nums -eq $expected ] ||
+               error "'$cmd' wrong: found $nums, expected $expected"
+
+       cmd="$LFS find $dir ! -newer$x$y $negref"
+       nums=$(eval $cmd | wc -l)
+       [ $nums -eq $expected ] ||
+               error "'$cmd' wrong: found $nums, expected $expected"
+
+       cmd="$LFS find $dir -newer$x$y $ref ! -newer$x$y $negref"
+       nums=$(eval $cmd | wc -l)
+       [ $nums -eq $expected ] ||
+               error "'$cmd' wrong: found $nums, expected $expected"
+
+       rm -rf $DIR/*
+}
+
+test_56oc() {
+       test_newerXY_base "a" "a"
+       test_newerXY_base "a" "m"
+       test_newerXY_base "a" "c"
+       test_newerXY_base "m" "a"
+       test_newerXY_base "m" "m"
+       test_newerXY_base "m" "c"
+       test_newerXY_base "c" "a"
+       test_newerXY_base "c" "m"
+       test_newerXY_base "c" "c"
+       test_newerXY_base "a" "t"
+       test_newerXY_base "m" "t"
+       test_newerXY_base "c" "t"
+}
+run_test 56oc "check lfs find -newerXY work"
+
 test_56p() {
        [ $RUNAS_ID -eq $UID ] &&
                skip_env "RUNAS_ID = UID = $UID -- skipping"
index 6160ea4..dc1f0e6 100644 (file)
@@ -457,6 +457,7 @@ command_t cmdlist[] = {
         "usage: find <directory|filename> ...\n"
         "     [[!] --atime|-A [+-]N[smhdwy]] [[!] --ctime|-C [+-]N[smhdwy]]\n"
         "     [[!] --mtime|-M [+-]N[smhdwy]] [[!] --blocks|-b N]\n"
+        "     [[!] --newer[XY] <reference>]\n"
         "     [--maxdepth|-D N] [[!] --mdt-index|--mdt|-m <uuid|index,...>]\n"
         "     [[!] --name|-n <pattern>] [[!] --ost|-O <uuid|index,...>]\n"
         "     [--print|-P] [--print0|-0] [[!] --size|-s [+-]N[bkMGTPE]]\n"
@@ -2976,6 +2977,7 @@ enum {
        LFS_MIRROR_INDEX_OPT,
        LFS_LAYOUT_FOREIGN_OPT,
        LFS_MODE_OPT,
+       LFS_NEWERXY_OPT,
 };
 
 /* functions */
@@ -4125,6 +4127,32 @@ static int lfs_find(int argc, char **argv)
                        .name = "mirror-state", .has_arg = required_argument },
        { .val = LFS_LAYOUT_FOREIGN_OPT,
                        .name = "foreign",      .has_arg = optional_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newer",        .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "neweraa",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "neweram",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newerac",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newerma",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newermm",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newermc",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newerca",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newercm",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newercc",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newerat",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newermt",      .has_arg = required_argument},
+       { .val = LFS_NEWERXY_OPT,
+                       .name = "newerct",      .has_arg = required_argument},
        { .val = 'c',   .name = "stripe-count", .has_arg = required_argument },
        { .val = 'c',   .name = "stripe_count", .has_arg = required_argument },
        { .val = 'C',   .name = "ctime",        .has_arg = required_argument },
@@ -4176,6 +4204,7 @@ static int lfs_find(int argc, char **argv)
 /* getstripe { .val = 'v', .name = "verbose",  .has_arg = no_argument }, */
 /* getstripe { .val = 'y', .name = "yaml",     .has_arg = no_argument }, */
        { .name = NULL } };
+       int optidx = 0;
        int pathstart = -1;
        int pathend = -1;
        int pathbad = -1;
@@ -4190,7 +4219,7 @@ static int lfs_find(int argc, char **argv)
        /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */
        while ((c = getopt_long_only(argc, argv,
                        "-0A:b:c:C:D:E:g:G:H:i:L:m:M:n:N:O:Ppqrs:S:t:T:u:U:vz:",
-                       long_opts, NULL)) >= 0) {
+                       long_opts, &optidx)) >= 0) {
                 xtime = NULL;
                 xsign = NULL;
                 if (neg_opt)
@@ -4412,6 +4441,130 @@ static int lfs_find(int argc, char **argv)
                        param.fp_exclude_foreign = !!neg_opt;
                        break;
                }
+               case LFS_NEWERXY_OPT: {
+                       char x = 'm';
+                       char y = 'm';
+                       int xidx;
+                       int negidx;
+                       time_t *newery;
+                       time_t ref = time(NULL);
+
+                       /* no need to check bad options, they won't get here */
+                       if (strlen(long_opts[optidx].name) == 7) {
+                               x = long_opts[optidx].name[5];
+                               y = long_opts[optidx].name[6];
+                       }
+
+                       if (y == 't') {
+                               static const char *const fmts[] = {
+                                       "%Y-%m-%d %H:%M:%S",
+                                       "%Y-%m-%d %H:%M",
+                                       "%Y-%m-%d",
+                                       "%H:%M:%S", /* sometime today */
+                                       "%H:%M",
+                                       "@%s",
+                                       "%s",
+                                       NULL };
+                               struct tm tm;
+                               bool found = false;
+                               int i;
+
+                               for (i = 0; fmts[i] != NULL; i++) {
+                                       char *ptr;
+
+                                       /* Init for times relative to today */
+                                       if (strncmp(fmts[i], "%H", 2) == 0)
+                                               localtime_r(&ref, &tm);
+                                       else
+                                               memset(&tm, 0, sizeof(tm));
+                                       ptr = strptime(optarg, fmts[i], &tm);
+                                       /* Skip spaces */
+                                       while (ptr && isspace(*ptr))
+                                               ptr++;
+                                       if (ptr == optarg + strlen(optarg)) {
+                                               found = true;
+                                               break;
+                                       }
+                               }
+
+                               if (!found) {
+                                       fprintf(stderr,
+                                               "%s: invalid time '%s'\n",
+                                               progname, optarg);
+                                       fprintf(stderr,
+                                               "supported formats are:\n  ");
+                                       for (i = 0; fmts[i] != NULL; i++)
+                                               fprintf(stderr, "'%s', ",
+                                                       fmts[i]);
+                                       fprintf(stderr, "\n");
+                                       ret = -EINVAL;
+                                       goto err;
+                               }
+
+                               ref = mktime(&tm);
+                       } else {
+                               struct stat statbuf;
+
+                               if (stat(optarg, &statbuf) < 0) {
+                                       fprintf(stderr,
+                                               "%s: cannot stat file '%s': %s\n",
+                                               progname, optarg,
+                                               strerror(errno));
+                                       ret = -errno;
+                                       goto err;
+                               }
+
+                               switch (y) {
+                               case 'a':
+                                       ref = statbuf.st_atime;
+                                       break;
+                               case 'm':
+                                       ref = statbuf.st_mtime;
+                                       break;
+                               case 'c':
+                                       ref = statbuf.st_ctime;
+                                       break;
+                               default:
+                                       fprintf(stderr,
+                                               "%s: invalid Y argument: '%c'\n",
+                                               progname, x);
+                                       ret = -EINVAL;
+                                       goto err;
+                               }
+                       }
+
+                       switch (x) {
+                       case 'a':
+                               xidx = NEWERXY_ATIME;
+                               break;
+                       case 'm':
+                               xidx = NEWERXY_MTIME;
+                               break;
+                       case 'c':
+                               xidx = NEWERXY_CTIME;
+                               break;
+                       default:
+                               fprintf(stderr,
+                                       "%s: invalid X argument: '%c'\n",
+                                       progname, x);
+                               ret = -EINVAL;
+                               goto err;
+                       }
+
+                       negidx = !!neg_opt;
+                       newery = &param.fp_newery[xidx][negidx];
+
+                       if (*newery == 0) {
+                               *newery = ref;
+                       } else {
+                               if (negidx)
+                                       *newery = *newery > ref ? ref : *newery;
+                               else
+                                       *newery = *newery > ref ? *newery : ref;
+                       }
+                       param.fp_newerxy = 1;
+                       break;
+               }
                case 'g':
                case 'G':
                        rc = name2gid(&param.fp_gid, optarg);
index 77998f4..4238208 100644 (file)
@@ -4044,6 +4044,57 @@ static int find_time_check(lstatx_t *stx, struct find_param *param, int mds)
        return rc;
 }
 
+static int find_newerxy_check(lstatx_t *stx, struct find_param *param, int mds)
+{
+       int i;
+       int rc = 1;
+       int rc2;
+
+       for (i = 0; i < 2; i++) {
+               /* Check if file is accepted. */
+               if (param->fp_newery[NEWERXY_ATIME][i]) {
+                       rc2 = find_value_cmp(stx->stx_atime.tv_sec,
+                                            param->fp_newery[NEWERXY_ATIME][i],
+                                            -1, i, 0, mds);
+                       if (rc2 < 0)
+                               return rc2;
+                       rc = rc2;
+               }
+
+               if (param->fp_newery[NEWERXY_MTIME][i]) {
+                       rc2 = find_value_cmp(stx->stx_mtime.tv_sec,
+                                            param->fp_newery[NEWERXY_MTIME][i],
+                                            -1, i, 0, mds);
+                       if (rc2 < 0)
+                               return rc2;
+
+                       /*
+                        * If the previous check matches, but this one is not
+                        * yet clear, we should return 0 to do an RPC on OSTs.
+                        */
+                       if (rc == 1)
+                               rc = rc2;
+               }
+
+               if (param->fp_newery[NEWERXY_CTIME][i]) {
+                       rc2 = find_value_cmp(stx->stx_ctime.tv_sec,
+                                            param->fp_newery[NEWERXY_CTIME][i],
+                                            -1, i, 0, mds);
+                       if (rc2 < 0)
+                               return rc2;
+
+                       /*
+                        * If the previous check matches, but this one is not
+                        * yet clear, we should return 0 to do an RPC on OSTs.
+                        */
+                       if (rc == 1)
+                               rc = rc2;
+               }
+       }
+
+       return rc;
+}
+
 /**
  * Check whether the stripes matches the indexes user provided
  *       1   : matched
@@ -4555,7 +4606,7 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp,
        /* Request MDS for the stat info if some of these parameters need
         * to be compared. */
        if (param->fp_obd_uuid || param->fp_mdt_uuid ||
-           param->fp_check_uid || param->fp_check_gid ||
+           param->fp_check_uid || param->fp_check_gid || param->fp_newerxy ||
            param->fp_atime || param->fp_mtime || param->fp_ctime ||
            param->fp_check_size || param->fp_check_blocks ||
            find_check_lmm_info(param) ||
@@ -4835,12 +4886,22 @@ obd_matches:
                 int for_mds;
 
                for_mds = lustre_fs ?
-                       (S_ISREG(stx->stx_mode) && stripe_count) : 0;
+                         (S_ISREG(stx->stx_mode) && stripe_count) : 0;
                decision = find_time_check(stx, param, for_mds);
                if (decision == -1)
                        goto decided;
        }
 
+       if (param->fp_newerxy) {
+               int for_mds;
+
+               for_mds = lustre_fs ?
+                         (S_ISREG(stx->stx_mode) && stripe_count) : 0;
+               decision = find_newerxy_check(stx, param, for_mds);
+               if (decision == -1)
+                       goto decided;
+       }
+
        flags = param->fp_lmd->lmd_flags;
        if (param->fp_check_size &&
            ((S_ISREG(stx->stx_mode) && stripe_count) ||
@@ -4899,6 +4960,13 @@ obd_matches:
                decision = find_time_check(stx, param, 0);
                if (decision == -1)
                        goto decided;
+
+               if (param->fp_newerxy) {
+                       decision = find_newerxy_check(stx, param, 0);
+                       if (decision == -1)
+                               goto decided;
+
+               }
        }
 
        if (param->fp_check_size) {