From 00141b1a746d4733c2f52c7a7edec36da4cedcac Mon Sep 17 00:00:00 2001 From: Yang Sheng Date: Fri, 10 Jul 2020 23:31:17 +0800 Subject: [PATCH] LU-11776 utils: add support lfs find with mdt hash flag The lfs find can use mdt hash flag as a condition. Also change it can find with one more mdt hash type. Signed-off-by: Yang Sheng Change-Id: I599bb1a3cc2c9ea2a523f50f119bd93a5520d213 Reviewed-on: https://review.whamcloud.com/39340 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Lai Siyao Reviewed-by: Oleg Drokin --- lustre/doc/lfs-find.1 | 4 +- lustre/include/lustre/lustreapi.h | 4 +- lustre/include/uapi/linux/lustre/lustre_user.h | 13 +++-- lustre/lod/lproc_lod.c | 6 -- lustre/mdd/mdd_dir.c | 5 -- lustre/tests/sanity-lfsck.sh | 7 +++ lustre/tests/sanity.sh | 8 ++- lustre/utils/lfs.c | 76 +++++++++++++++++++++++--- lustre/utils/liblustreapi.c | 34 ++++++++---- 9 files changed, 114 insertions(+), 43 deletions(-) diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 0cb2b8a..7598f26 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -12,8 +12,6 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--component-flags|\fB--comp-flags\fR <[^]\fIflag\fB,\fR...>] [[\fB!\fR] \fB--component-start|\fB--comp-start\fR [\fB+-\fR]\fIn\fR[\fBKMGTPE\fR]] [[\fB!\fR] \fB--extension-size|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]] - [[\fB!\fR] \fB--gid\fR|\fB-g\fR|\fB--group\fR|\fB-G\fR <\fIgname\fR>|<\fIgid\fR>] - [[\fB!\fR] \fB--layout\fR|\fB-L mdt\fR,\fBraid0\fR,\fBreleased\fR] [[\fB!\fR] \fB--foreign\fR [<\fItype\fR>]] [[\fB!\fR] \fB--gid\fR|\fB-g\fR|\fB--group\fR|\fB-G\fR <\fIgname\fR>|<\fIgid\fR>] [[\fB!\fR] \fB--layout\fR|\fB-L mdt\fR,\fBraid0\fR,\fBreleased\fR] @@ -21,7 +19,7 @@ lfs-find \- Lustre client utility to list files with specific attributes [\fB--maxdepth\fR|\fB-D\fI n\fR] [[\fB!\fR] \fB--mdt\fR|\fB--mdt-index\fR|\fB-m\fR <\fIuuid\fR|\fIindex\fR,...>] [[\fB!\fR] \fB--mdt-count\fR|\fB-T\fR [\fB+-\fR]\fIn\fR] -[[\fB!\fR] \fB--mdt-hash\fR|\fB-H <\fIhashtype\fR>] +[[\fB!\fR] \fB--mdt-hash\fR|\fB-H \fR<[^]\fIhashflag\fR,[^]\fIhashtype\fR,...>] [[\fB!\fR] \fB--mirror-count|\fB-N\fR [\fB+-\fR]\fIn\fR] [[\fB!\fR] \fB--mirror-state\fR <[^]\fIstate\fR>] [[\fB!\fR] \fB--mtime\fR|\fB-M\fR [\fB-+\fR]\fIn[smhdwy]\fR] diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 2619002..0b32875 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -287,7 +287,7 @@ struct find_param { fp_exclude_mirror_index:1, fp_check_mdt_count:1, fp_exclude_mdt_count:1, - fp_check_hash_type:1, + fp_check_hash_flag:1, fp_exclude_hash_type:1, fp_yaml:1, /* output layout in YAML */ fp_check_blocks:1, @@ -373,6 +373,8 @@ struct find_param { time_t fp_btime; int fp_bsign; + unsigned int fp_hash_inflags; + unsigned int fp_hash_exflags; }; int llapi_ostlist(char *path, struct find_param *param); diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index e67db9f..4d697d6 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -997,11 +997,14 @@ enum lmv_hash_type { LMV_HASH_TYPE_MAX, }; -#define LMV_HASH_TYPE_DEFAULT LMV_HASH_TYPE_FNV_1A_64 +static __attribute__((unused)) const char *mdt_hash_name[] = { + "none", + "all_char", + "fnv_1a_64", + "crush", +}; -#define LMV_HASH_NAME_ALL_CHARS "all_char" -#define LMV_HASH_NAME_FNV_1A_64 "fnv_1a_64" -#define LMV_HASH_NAME_CRUSH "crush" +#define LMV_HASH_TYPE_DEFAULT LMV_HASH_TYPE_FNV_1A_64 /* Right now only the lower part(0-16bits) of lmv_hash_type is being used, * and the higher part will be the flag to indicate the status of object, @@ -1061,8 +1064,6 @@ static inline bool lmv_hash_is_layout_changing(__u32 hash) lmv_hash_is_migrating(hash); } -extern char *mdt_hash_name[LMV_HASH_TYPE_MAX]; - struct lustre_foreign_type { __u32 lft_type; const char *lft_name; diff --git a/lustre/lod/lproc_lod.c b/lustre/lod/lproc_lod.c index 20aeafa..71eea45 100644 --- a/lustre/lod/lproc_lod.c +++ b/lustre/lod/lproc_lod.c @@ -977,12 +977,6 @@ static ssize_t lmv_failout_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_RW_ATTR(lmv_failout); -char *mdt_hash_name[] = { "none", - LMV_HASH_NAME_ALL_CHARS, - LMV_HASH_NAME_FNV_1A_64, - LMV_HASH_NAME_CRUSH, -}; - static ssize_t mdt_hash_show(struct kobject *kobj, struct attribute *attr, char *buf) { diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index 9c57a46..ce98d97 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -4081,11 +4081,6 @@ static int mdd_migrate_cmd_check(struct mdd_device *mdd, { __u32 lum_stripe_count = lum->lum_stripe_count; __u32 lmv_hash_type = lmv->lmv_hash_type; - char *mdt_hash_name[] = { "none", - LMV_HASH_NAME_ALL_CHARS, - LMV_HASH_NAME_FNV_1A_64, - LMV_HASH_NAME_CRUSH, - }; if (!lmv_is_sane(lmv)) return -EBADF; diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index 4360b27..4549a91 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -4848,6 +4848,10 @@ test_31a() { [ $repaired -ge 1 ] || error "(5) Fail to repair bad name hash: $repaired" + local rc=$($LFS find -H badtype $DIR/$tdir/striped_dir | wc -l) + [ $rc -ge 1 ] || + error "Fail to find flag bad type: $rc" + umount_client $MOUNT || error "(6) umount failed" mount_client $MOUNT || error "(7) mount failed" @@ -4951,6 +4955,9 @@ test_31c() { [ $repaired -eq 1 ] || error "(4) Fail to re-generate master LMV EA: $repaired" + local rc=$($LFS find -H lostlmv $DIR/$tdir/striped_dir | wc -l) + [ $rc -eq 1 ] || error "Fail to find flag lost LMV: $rc" + umount_client $MOUNT || error "(5) umount failed" mount_client $MOUNT || error "(6) mount failed" diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index dbd3dab..e4c4e0c 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -21936,15 +21936,17 @@ test_300i() { error "create dir0 under hash dir failed" $LFS setdirstripe -i0 -c$MDSCOUNT -H fnv_1a_64 $DIR/$tdir/hashdir/d1 || error "create dir1 under hash dir failed" + $LFS setdirstripe -i0 -c$MDSCOUNT -H crush $DIR/$tdir/hashdir/d2 || + error "create dir2 under hash dir failed" # unfortunately, we need to umount to clear dir layout cache for now # once we fully implement dir layout, we can drop this umount_client $MOUNT || error "umount failed" mount_client $MOUNT || error "mount failed" - $LFS find -H fnv_1a_64 $DIR/$tdir/hashdir - local dircnt=$($LFS find -H fnv_1a_64 $DIR/$tdir/hashdir | wc -l) - [ $dircnt -eq 1 ] || error "lfs find striped dir got:$dircnt,except:1" + $LFS find -H fnv_1a_64,crush $DIR/$tdir/hashdir + local dircnt=$($LFS find -H fnv_1a_64,crush $DIR/$tdir/hashdir | wc -l) + [ $dircnt -eq 2 ] || error "lfs find striped dir got:$dircnt,except:1" #set the stripe to be unknown hash type #define OBD_FAIL_UNKNOWN_LMV_STRIPE 0x1901 diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 42a26ab..f4330531 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -508,14 +508,12 @@ command_t cmdlist[] = { " [[!] --mirror-count|-N [+-]]\n" " [[!] --mirror-state <[^]state>]\n" " [[!] --mdt-count|-T [+-]]\n" - " [[!] --mdt-hash|-H \n" + " [[!] --mdt-hash|-H <[^][blm],[^]fnv_1a_64,all_char,crush,...>\n" " [[!] --mdt-index|-m ]\n" "\t !: used before an option indicates 'NOT' requested attribute\n" "\t -: used before a value indicates less than requested value\n" "\t +: used before a value indicates more than requested value\n" - "\thashtype: hash type of the striped directory.\n" - "\t fnv_1a_64 FNV-1a hash algorithm\n" - "\t all_char sum of characters % MDT_COUNT\n"}, + "\t ^: used before a flag indicates to exclude it\n"}, {"check", lfs_check, 0, "Display the status of MGTs, MDTs or OSTs (as specified in the command)\n" "or all the servers (MGTs, MDTs and OSTs).\n" @@ -1488,6 +1486,65 @@ out_free: return rc; } +static int mdthash_input(char *string, __u32 *inflags, + __u32 *exflags, __u32 *type) +{ + char *name; + struct mhf_list { + char *name; + __u32 flag; + } mhflist[] = { + {"migrating", LMV_HASH_FLAG_MIGRATION}, + {"badtype", LMV_HASH_FLAG_BAD_TYPE}, + {"lostlmv", LMV_HASH_FLAG_LOST_LMV}, + }; + + if (string == NULL) + return -EINVAL; + + *inflags = 0; + *exflags = 0; + *type = 0; + for (name = strtok(string, ","); name; name = strtok(NULL, ",")) { + bool found = false; + int i; + + for (i = 0; i < ARRAY_SIZE(mhflist); i++) { + if (strcmp(name, mhflist[i].name) == 0 || + name[0] == mhflist[i].name[0]) { + *inflags |= mhflist[i].flag; + found = true; + } else if (name[0] == '^' && + (strcmp(name + 1, mhflist[i].name) == 0 || + name[1] == mhflist[i].name[0])) { + *exflags |= mhflist[i].flag; + found = true; + } + } + if (!found) { + i = check_hashtype(name); + if (i > 0) { + *type |= 1 << i; + continue; + } + llapi_printf(LLAPI_MSG_ERROR, + "%s: invalid mdt_hash value '%s'\n", + progname, name); + return -EINVAL; + } + } + + /* don't allow to include and exclude the same flag */ + if (*inflags & *exflags) { + llapi_printf(LLAPI_MSG_ERROR, + "%s: include and exclude same flag '%s'\n", + progname, string); + return -EINVAL; + } + + return 0; +} + static int mirror_str2state(char *string, __u16 *state, __u16 *neg_state) { if (!string) @@ -5042,14 +5099,15 @@ static int lfs_find(int argc, char **argv) param.fp_check_gid = 1; break; case 'H': - param.fp_hash_type = check_hashtype(optarg); - if (param.fp_hash_type == 0) { - fprintf(stderr, "error: bad hash_type '%s'\n", - optarg); + rc = mdthash_input(optarg, ¶m.fp_hash_inflags, + ¶m.fp_hash_exflags, + ¶m.fp_hash_type); + if (rc) { ret = -1; goto err; } - param.fp_check_hash_type = 1; + if (param.fp_hash_inflags || param.fp_hash_exflags) + param.fp_check_hash_flag = 1; param.fp_exclude_hash_type = !!neg_opt; break; case 'l': diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 0278d4c..7e2ffd9 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -86,12 +86,6 @@ static int llapi_msg_level = LLAPI_MSG_MAX; const char *liblustreapi_cmd; -char *mdt_hash_name[] = { "none", - LMV_HASH_NAME_ALL_CHARS, - LMV_HASH_NAME_FNV_1A_64, - LMV_HASH_NAME_CRUSH, -}; - struct lustre_foreign_type lu_foreign_types[] = { {.lft_type = LU_FOREIGN_TYPE_NONE, .lft_name = "none"}, {.lft_type = LU_FOREIGN_TYPE_SYMLINK, .lft_name = "symlink"}, @@ -4751,7 +4745,8 @@ static int cb_find_init(char *path, int p, int *dp, param->fp_atime || param->fp_mtime || param->fp_ctime || param->fp_check_size || param->fp_check_blocks || find_check_lmm_info(param) || - param->fp_check_mdt_count || param->fp_check_hash_type) + param->fp_check_mdt_count || param->fp_hash_type || + param->fp_check_hash_flag) decision = 0; if (param->fp_type != 0 && checked_type == 0) @@ -4759,7 +4754,8 @@ static int cb_find_init(char *path, int p, int *dp, if (decision == 0) { if (d != -1 && (param->fp_check_mdt_count || - param->fp_check_hash_type || param->fp_check_foreign)) { + param->fp_hash_type || param->fp_check_foreign || + param->fp_check_hash_flag)) { param->fp_get_lmv = 1; ret = cb_get_dirstripe(path, &d, param); if (ret != 0) { @@ -4923,15 +4919,17 @@ static int cb_find_init(char *path, int p, int *dp, goto decided; } - if (param->fp_check_hash_type) { + if (param->fp_hash_type) { __u32 found; + __u32 type = param->fp_lmv_md->lum_hash_type & + LMV_HASH_TYPE_MASK; if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { decision = -1; goto decided; } - found = param->fp_lmv_md->lum_hash_type & param->fp_hash_type; + found = (1 << type) & param->fp_hash_type; if ((found && param->fp_exclude_hash_type) || (!found && !param->fp_exclude_hash_type)) { decision = -1; @@ -4939,6 +4937,22 @@ static int cb_find_init(char *path, int p, int *dp, } } + if (param->fp_check_hash_flag) { + __u32 flags = param->fp_lmv_md->lum_hash_type & + ~LMV_HASH_TYPE_MASK; + + if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { + decision = -1; + goto decided; + } + + if (!(flags & param->fp_hash_inflags) || + (flags & param->fp_hash_exflags)) { + decision = -1; + goto decided; + } + } + /* If an OBD UUID is specified but none matches, skip this file. */ if ((param->fp_obd_uuid && param->fp_obd_index == OBD_NOT_FOUND) || (param->fp_mdt_uuid && param->fp_mdt_index == OBD_NOT_FOUND)) -- 1.8.3.1