Whamcloud - gitweb
LU-16760 utils: support 'lfs find --attrs' and '-printf %La' 62/51562/8
authorSebastien Buisson <sbuisson@ddn.com>
Tue, 4 Jul 2023 07:28:37 +0000 (09:28 +0200)
committerOleg Drokin <green@whamcloud.com>
Thu, 24 Aug 2023 04:33:54 +0000 (04:33 +0000)
Add support to "lfs find" to filter on file attribute flags, with the
syntax "[!] --attrs=[^]ATTR[,...]".
Add support to "lfs find" to print file attribute flags with
"-printf %La".

Add sanity-sec test_65 for Encrypted and Immutable flags.

Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I5e5cfe5c8c8cbed8bb79f3ad6d8116347ecfe6ac
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51562
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Zhenyu Xu <bobijam@hotmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
lustre/doc/lfs-find.1
lustre/include/lustre/lustreapi.h
lustre/include/uapi/linux/lustre/lustre_user.h
lustre/llite/dir.c
lustre/tests/sanity-sec.sh
lustre/utils/lfs.c
lustre/utils/liblustreapi.c

index d78e6af..f0be1d8 100644 (file)
@@ -4,6 +4,7 @@ lfs-find \- Lustre client utility to list files with specific attributes
 .SH SYNOPSIS
 .B lfs find \fR<\fIdirectory\fR|\fIfilename \fR...>
       [[\fB!\fR] \fB--atime\fR|\fB-A\fR [\fB-+\fR]\fIn[smhdwy]\fR]
+[[\fB!\fR] \fB--attrs\fR=\fI[^]ATTR[,...]\fR]
 [[\fB!\fR] \fB--blocks\fR|\fB-b\fR [\fB+-\fR]\fIn\fR]
       [[\fB!\fR] \fB--btime\fR|\fB-B\fR [\fB+-\fR]\fIn[smhdwy]\fR]
 [[\fB!\fR] \fB--ctime\fR|\fB-C\fR [\fB+-\fR]\fIn[smhdwy]\fR]
@@ -63,6 +64,11 @@ has a margin of error of one day, while
 .B -atime 72h
 has a margin of error of one hour.
 .TP
+.BR --attrs
+File has ATTRS attribute flags. Supported attributes are (non exhaustive list):
+Compressed (c), Immutable (i), Append_Only (a), No_Dump (d), Encrypted (E),
+Automount (M)
+.TP
 .BR --blocks | -b
 Blocks allocated by the file is \fIn\fR Kibibytes (if no units are given),
 \fIn\fR 512-byte \fBb\fRlocks, or \fBK\fRibi-, \fBM\fRebi-, \fBG\fRibi-,
@@ -365,6 +371,16 @@ s=socket l=symbolic link)
 .TP
 Lustre-specific information about a file can be printed using these directives:
 .TP
+.B %La
+Comma-separated list of file's named attribute flags in short form (letter), or
+hex value of any unknown attributes.
+.RE
+.TP
+.B %LA
+Comma-separated list of file's named attribute flags, or hex value of any
+unknown attributes.
+.RE
+.TP
 .B %Lc
 File\'s stripe count.  For a composite file, this is the stripe count of the last
 instantiated component.
@@ -395,7 +411,6 @@ Numeric project ID assigned to the file or directory.
 .B %LS
 File's stripe size.  For a composite file, this is the stripe size of the last
 instantiated component.
-.RE
 .TP
 .BR --projid
 File has specified numeric project ID.
index 8a23b9b..a1a90f2 100644 (file)
@@ -347,8 +347,8 @@ struct find_param {
                                 fp_exclude_perm:1,
                                 fp_stop_on_error:1, /* stop iteration on error */
                                 fp_exclude_nlink:1, /* Once used, we must add*/
-                                fp_unused_bit6:1, /* a separate flag field at*/
-                                fp_unused_bit7:1; /* the end of the struct.  */
+                                fp_exclude_attrs:1, /* a separate flag field */
+                                fp_unused_bit7:1;   /* at end of struct.  */
 
        enum llapi_layout_verbose fp_verbose;
        int                      fp_quiet;
@@ -424,6 +424,8 @@ struct find_param {
        /* Print all information (lfs find only) */
        char                     *fp_format_printf_str;
        nlink_t                  fp_nlink;
+       __u64                    fp_attrs;
+       __u64                    fp_neg_attrs;
 };
 
 int llapi_ostlist(char *path, struct find_param *param);
@@ -1157,6 +1159,23 @@ static const struct hsm_flag_name {
        { HS_PCCRO,     "pccro" },
 };
 
+/* Currently known file attributes.
+ * Update if more attributes are added in lustre_user.h.
+ */
+static const struct attrs_name {
+       uint64_t         an_attr;
+       const char      *an_name;
+       const char       an_shortname;
+} attrs_array[] = {
+       { STATX_ATTR_COMPRESSED, "Compressed",  'c' },
+       { STATX_ATTR_IMMUTABLE,  "Immutable",   'i' },
+       { STATX_ATTR_APPEND,     "Append_Only", 'a' },
+       { STATX_ATTR_NODUMP,     "No_Dump",     'd' },
+       { STATX_ATTR_ENCRYPTED,  "Encrypted",   'E' },
+       { STATX_ATTR_AUTOMOUNT,  "Automount",   'M' },
+       { 0,                     NULL,           0  }
+};
+
 /**
  * Gets the attribute flags of the current component.
  */
index 3bdae7f..48ead30 100644 (file)
@@ -238,6 +238,7 @@ struct statx {
 #define STATX_ATTR_ENCRYPTED           0x00000800 /* [I] File requires key to decrypt in fs */
 
 #define STATX_ATTR_AUTOMOUNT           0x00001000 /* Dir: Automount trigger */
+/* Update attrs_array in lustreapi.h if new attributes are added. */
 
 #define AT_STATX_SYNC_TYPE     0x6000  /* Type of synchronisation required from statx() */
 #define AT_STATX_SYNC_AS_STAT  0x0000  /* - Do whatever stat() does */
index 342c65f..8e1a1fb 100644 (file)
@@ -2004,6 +2004,20 @@ out_rmdir:
                        stx.stx_dev_minor = MINOR(inode->i_sb->s_dev);
                        stx.stx_mask |= STATX_BASIC_STATS | STATX_BTIME;
 
+                       stx.stx_attributes_mask = STATX_ATTR_IMMUTABLE |
+                                                 STATX_ATTR_APPEND;
+#ifdef HAVE_LUSTRE_CRYPTO
+                       stx.stx_attributes_mask |= STATX_ATTR_ENCRYPTED;
+#endif
+                       if (body->mbo_valid & OBD_MD_FLFLAGS) {
+                               stx.stx_attributes |= body->mbo_flags;
+                               /* if Lustre specific LUSTRE_ENCRYPT_FL flag is
+                                * set, also set ext4 equivalent to please statx
+                                */
+                               if (body->mbo_flags & LUSTRE_ENCRYPT_FL)
+                                    stx.stx_attributes |= STATX_ATTR_ENCRYPTED;
+                       }
+
                        /* For a striped directory, the size and blocks returned
                         * from MDT is not correct.
                         * The size and blocks are aggregated by client across
index 4b2b4a4..9f0d9df 100755 (executable)
@@ -5827,6 +5827,115 @@ test_64f() {
 }
 run_test 64f "Nodemap enforces fscrypt_admin RBAC roles"
 
+look_for_files() {
+       local pattern=$1
+       local neg=$2
+       local path=$3
+       local expected=$4
+       local res
+
+       (( neg == 1 )) || neg=""
+       $LFS find -type f ${neg:+"!"} --attrs $pattern $path > $TMP/res
+       cat $TMP/res
+       res=$(cat $TMP/res | wc -l)
+       (( res == $expected )) ||
+               error "Find $pattern $path: found $res, expected $expected"
+}
+
+test_65() {
+       local dirbis=$DIR/${tdir}_bis
+       local testfile=$DIR/$tdir/$tfile
+       local res
+
+       $LCTL get_param mdc.*.import | grep -q client_encryption ||
+               skip "client encryption not supported"
+
+       mount.lustre --help |& grep -q "test_dummy_encryption:" ||
+               skip "need dummy encryption support"
+
+       # $dirbis is not going to be encrypted, as client
+       # is not mounted with -o test_dummy_encryption yet
+       mkdir $dirbis
+       stack_trap "rm -rf $dirbis" EXIT
+       touch $dirbis/$tfile.1
+       touch $dirbis/$tfile.2
+       chattr +i $dirbis/$tfile.2
+       stack_trap "chattr -i $dirbis/$tfile.2" EXIT
+
+       stack_trap cleanup_for_enc_tests EXIT
+       setup_for_enc_tests
+
+       # All files/dirs under $DIR/$tdir are encrypted
+       touch $testfile.1
+       touch $testfile.2
+       chattr +i $testfile.2
+       stack_trap "chattr -i $testfile.2" EXIT
+
+       $LFS find -printf "%p %LA\n" $dirbis/$tfile.1
+       res=$($LFS find -printf "%LA" $dirbis/$tfile.1)
+       [ "$res" == "---" ] ||
+               error "$dirbis/$tfile.1 should have no attr, showed $res (1)"
+       $LFS find -printf "%p %La\n" $dirbis/$tfile.1
+       res=$($LFS find -printf "%La" $dirbis/$tfile.1)
+       [ "$res" == "---" ] ||
+               error "$dirbis/$tfile.1 should have no attr, showed $res (2)"
+       $LFS find -printf "%p %LA\n" $dirbis/$tfile.2
+       res=$($LFS find -printf "%LA" $dirbis/$tfile.2)
+       [ "$res" == "Immutable" ] ||
+               error "$dirbis/$tfile.2 should be Immutable, showed $res"
+       $LFS find -printf "%p %La\n" $dirbis/$tfile.2
+       res=$($LFS find -printf "%La" $dirbis/$tfile.2)
+       [ "$res" == "i" ] ||
+               error "$dirbis/$tfile.2 should be 'i', showed $res"
+       $LFS find -printf "%p %LA\n" $testfile.1
+       res=$($LFS find -printf "%LA" $testfile.1)
+       [ "$res" == "Encrypted" ] ||
+               error "$testfile.1 should be Encrypted, showed $res"
+       $LFS find -printf "%p %La\n" $testfile.1
+       res=$($LFS find -printf "%La" $testfile.1)
+       [ "$res" == "E" ] ||
+               error "$testfile.1 should be 'E', showed $res"
+       $LFS find -printf "%p %LA\n" $testfile.2
+       res=$($LFS find -printf "%LA" $testfile.2)
+       [ "$res" == "Immutable,Encrypted" ] ||
+               error "$testfile.2 should be Immutable,Encrypted, showed $res"
+       $LFS find -printf "%p %La\n" $testfile.2
+       res=$($LFS find -printf "%La" $testfile.2)
+       [ "$res" == "iE" ] ||
+               error "$testfile.2 should be 'iE', showed $res"
+
+       echo Expecting to find 2 encrypted files
+       look_for_files Encrypted 0 "$DIR/${tdir}*" 2
+       echo Expecting to find 2 encrypted files
+       look_for_files E 0 "$DIR/${tdir}*" 2
+
+       echo Expecting to find 2 non-encrypted files
+       look_for_files Encrypted 1 "$DIR/${tdir}*" 2
+       echo Expecting to find 2 non-encrypted files
+       look_for_files E 1 "$DIR/${tdir}*" 2
+
+       echo Expecting to find 1 encrypted+immutable file
+       look_for_files "Encrypted,Immutable" 0 "$DIR/${tdir}*" 1
+       echo Expecting to find 1 encrypted+immutable file
+       look_for_files "Ei" 0 "$DIR/${tdir}*" 1
+
+       echo Expecting to find 1 encrypted+^immutable file
+       look_for_files "Encrypted,^Immutable" 0 "$DIR/${tdir}*" 1
+       echo Expecting to find 1 encrypted+^immutable file
+       look_for_files "E^i" 0 "$DIR/${tdir}*" 1
+
+       echo Expecting to find 1 ^encrypted+immutable file
+       look_for_files "^Encrypted,Immutable" 0 "$DIR/${tdir}*" 1
+       echo Expecting to find 1 ^encrypted+immutable file
+       look_for_files "^Ei" 0 "$DIR/${tdir}*" 1
+
+       echo Expecting to find 1 ^encrypted+^immutable file
+       look_for_files "^Encrypted,^Immutable" 0 "$DIR/${tdir}*" 1
+       echo Expecting to find 1 ^encrypted+^immutable file
+       look_for_files "^E^i" 0 "$DIR/${tdir}*" 1
+}
+run_test 65 "lfs find -printf %La and --attrs support"
+
 log "cleanup: ======================================================"
 
 sec_unsetup() {
index e8695a5..6c439de 100644 (file)
@@ -404,6 +404,7 @@ command_t cmdlist[] = {
         "usage: find <directory|filename> ...\n"
         "     [[!] --atime|-A [+-]N[smhdwy]] [[!] --btime|-B [+-]N[smhdwy]]\n"
         "     [[!] --ctime|-C [+-]N[smhdwy]] [[!] --mtime|-M [+-]N[smhdwy]]\n"
+        "     [[!] --attrs=[^]ATTR[,...]]\n"
         "     [[!] --blocks|-b N] [[!] --component-count [+-]<comp_cnt>]\n"
         "     [[!] --component-start [+-]N[kMGTPE]]\n"
         "     [[!] --component-end|-E [+-]N[kMGTPE]]\n"
@@ -3512,7 +3513,8 @@ enum {
        LFS_HEX_IDX_OPT,
        LFS_STATS_OPT,
        LFS_STATS_INTERVAL_OPT,
-       LFS_LINKS_OPT
+       LFS_LINKS_OPT,
+       LFS_ATTRS_OPT
 };
 
 #ifndef LCME_USER_MIRROR_FLAGS
@@ -4790,6 +4792,63 @@ static int name2layout(__u32 *layout, char *name)
        return 0;
 }
 
+static int name2attrs(char *name, __u64 *attrs, __u64 *neg_attrs)
+{
+       char *ptr, *attr_name = name;
+       struct attrs_name *ap;
+       int islongopt = 0; /* 1 true; 0 not known yet; -1 false. */
+
+       *attrs = 0;
+       *neg_attrs = 0;
+
+       if (strchr(name, ','))
+               islongopt = 1;
+
+       for (ptr = name; ; ptr = NULL) {
+               if (islongopt != -1)
+                       attr_name = strtok(ptr, ",");
+               else
+                       attr_name = attr_name + 1;
+               if (!attr_name || *attr_name == '\0')
+                       break;
+
+               for (ap = (struct attrs_name *)attrs_array;
+                    ap->an_attr != 0;
+                    ap++) {
+                       if (islongopt != -1 &&
+                           strcmp(attr_name, ap->an_name) == 0) {
+                               *attrs |= ap->an_attr;
+                               islongopt = 1;
+                               break;
+                       } else if (islongopt != -1 && attr_name[0] == '^' &&
+                                  strcmp(attr_name + 1, ap->an_name) == 0) {
+                               *neg_attrs |= ap->an_attr;
+                               islongopt = 1;
+                               break;
+                       } else if (islongopt != 1 &&
+                                  *attr_name == ap->an_shortname) {
+                               *attrs |= ap->an_attr;
+                               islongopt = -1;
+                               break;
+                       } else if (islongopt != 1 && *attr_name == '^' &&
+                                  attr_name[1] == ap->an_shortname) {
+                               *neg_attrs |= ap->an_attr;
+                               islongopt = -1;
+                               attr_name++;
+                               break;
+                       }
+               }
+
+               if (ap->an_attr == 0) {
+                       /* provided attr is unknown */
+                       fprintf(stderr, "error: bad attribute name '%s'\n",
+                               attr_name);
+                       return -1;
+               }
+       }
+       return 0;
+}
+
 static int parse_symbolic(const char *input, mode_t *outmode, const char **end)
 {
        int loop;
@@ -5030,6 +5089,8 @@ static int lfs_find(int argc, char **argv)
        };
        struct option long_opts[] = {
        { .val = 'A',   .name = "atime",        .has_arg = required_argument },
+       { .val = LFS_ATTRS_OPT,
+                       .name = "attrs",        .has_arg = required_argument },
        { .val = 'b',   .name = "blocks",       .has_arg = required_argument },
        { .val = 'B',   .name = "btime",        .has_arg = required_argument },
        { .val = 'B',   .name = "Btime",        .has_arg = required_argument },
@@ -5252,6 +5313,13 @@ static int lfs_find(int argc, char **argv)
                        if (rc)
                                *xsign = rc;
                        break;
+               case LFS_ATTRS_OPT:
+                       ret = name2attrs(optarg, &param.fp_attrs,
+                                        &param.fp_neg_attrs);
+                       if (ret)
+                               goto err;
+                       param.fp_exclude_attrs = !!neg_opt;
+                       break;
                case 'b':
                        if (optarg[0] == '+') {
                                param.fp_blocks_sign = -1;
index 3c3615d..b405cd0 100644 (file)
@@ -1646,7 +1646,6 @@ again:
 static void convert_lmd_statx(struct lov_user_mds_data *lmd_v2, lstat_t *st,
                              bool strict)
 {
-       memset(&lmd_v2->lmd_stx, 0, sizeof(lmd_v2->lmd_stx));
        lmd_v2->lmd_stx.stx_blksize = st->st_blksize;
        lmd_v2->lmd_stx.stx_nlink = st->st_nlink;
        lmd_v2->lmd_stx.stx_uid = st->st_uid;
@@ -4545,6 +4544,26 @@ static int find_check_mirror_options(struct find_param *param)
        return ret;
 }
 
+static int find_check_attr_options(struct find_param *param)
+{
+       bool found = true;
+       __u64 attrs;
+
+       attrs = param->fp_lmd->lmd_stx.stx_attributes_mask &
+               param->fp_lmd->lmd_stx.stx_attributes;
+
+       /* This is a AND between all (negated) specified attributes */
+       if ((param->fp_attrs && (param->fp_attrs & attrs) != param->fp_attrs) ||
+           (param->fp_neg_attrs && (param->fp_neg_attrs & attrs)))
+               found = false;
+
+       if ((found && param->fp_exclude_attrs) ||
+           (!found && !param->fp_exclude_attrs))
+               return -1;
+
+       return 1;
+}
+
 static bool find_check_lmm_info(struct find_param *param)
 {
        return param->fp_check_pool || param->fp_check_stripe_count ||
@@ -4749,6 +4768,62 @@ format_done:
 }
 
 /*
+ * Print file attributes as a comma-separated list of named attribute flags,
+ * and hex value of any unknown attributes.
+ *
+ * @param[out] buffer  Location where file attributes are written
+ * @param[in]  size    Size of the available buffer.
+ * @pararm[in] stx     struct statx containing attributes to print
+ * @return             Number of bytes written to output buffer
+ */
+static int printf_format_file_attributes(char *buffer, size_t size,
+                                        lstatx_t stx, bool longopt)
+{
+       uint64_t attrs = stx.stx_attributes_mask & stx.stx_attributes;
+       int bytes = 0, wrote = 0, first = 1;
+       uint64_t known_attrs = 0;
+       struct attrs_name *ap;
+
+       /* before all, print '---' if no attributes, and exit */
+       if (!attrs) {
+               bytes = snprintf(buffer, size - wrote, "---");
+               wrote += bytes;
+               goto format_done;
+       }
+
+       /* first, browse list of known attributes */
+       for (ap = (struct attrs_name *)attrs_array; ap->an_attr != 0; ap++) {
+               known_attrs |= ap->an_attr;
+               if (attrs & ap->an_attr) {
+                       if (longopt)
+                               bytes = snprintf(buffer, size - wrote, "%s%s",
+                                                first ? "" : ",", ap->an_name);
+                       else
+                               bytes = snprintf(buffer, size - wrote, "%c",
+                                                ap->an_shortname);
+                       wrote += bytes;
+                       first = 0;
+                       if (wrote >= size)
+                               goto format_done;
+                       buffer += bytes;
+               }
+       }
+
+       /* second, print hex value for unknown attributes */
+       attrs &= ~known_attrs;
+       if (attrs) {
+               bytes = snprintf(buffer, size - wrote, "%s0x%lx",
+                                first ? "" : ",", attrs);
+               wrote += bytes;
+       }
+
+format_done:
+       if (wrote >= size)
+               wrote = size - 1;
+       return wrote;
+}
+
+/*
  * Parse Lustre-specific format sequences of the form %L{x}.
  *
  * @param[in]  seq     String being parsed for format sequence.  The leading
@@ -4776,6 +4851,7 @@ int printf_format_lustre(char *seq, char *buffer, size_t size, int *wrote,
        uint64_t str_cnt, str_size, idx;
        char pool_name[LOV_MAXPOOLNAME + 1] = { '\0' };
        int err, bytes, i;
+       bool longopt = true;
        int rc = 2;     /* all current valid sequences are 2 chars */
        *wrote = 0;
 
@@ -4805,6 +4881,14 @@ int printf_format_lustre(char *seq, char *buffer, size_t size, int *wrote,
                else
                        *wrote = snprintf(buffer, size, "%u", projid);
                goto format_done;
+       case 'a': /* file attributes */
+               longopt = false;
+               fallthrough;
+       case 'A':
+               *wrote = printf_format_file_attributes(buffer, size,
+                                                      param->fp_lmd->lmd_stx,
+                                                      longopt);
+               goto format_done;
        }
 
        /* Other formats for files/dirs need to be handled differently */
@@ -5224,7 +5308,7 @@ static int cb_find_init(char *path, int p, int *dp,
            find_check_lmm_info(param) ||
            param->fp_check_mdt_count || param->fp_hash_type ||
            param->fp_check_hash_flag || param->fp_perm_sign ||
-           param->fp_nlink ||
+           param->fp_nlink || param->fp_attrs || param->fp_neg_attrs ||
            gather_all)
                decision = 0;
 
@@ -5584,6 +5668,12 @@ obd_matches:
                }
        }
 
+       if (param->fp_attrs || param->fp_neg_attrs) {
+               decision = find_check_attr_options(param);
+               if (decision == -1)
+                       goto decided;
+       }
+
        flags = param->fp_lmd->lmd_flags;
        if (param->fp_check_size &&
            ((S_ISREG(lmd->lmd_stx.stx_mode) && stripe_count) ||
@@ -5930,7 +6020,7 @@ int validate_printf_fmt(char *c)
 {
        char *valid_fmt_single = "abcGkmnpstUwy%";
        char *valid_fmt_double = "ACTW";
-       char *valid_fmt_lustre = "cFhioPpS";
+       char *valid_fmt_lustre = "aAcFhioPpS";
        char curr = *c, next;
 
        if (curr == '\0') {