From 4063d21e395e16f486d2e3414d473c4376e9e7a4 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Fri, 28 May 2021 15:15:10 -0600 Subject: [PATCH] LU-14489 utils: fix 'lfs find --mdt-count' Running "lfs find --mdt-count" causes the find to exit if there is no directory striping, rather than continuing to the next item. If cb_get_dirstripe() receives ENODATA then it should consider that directory as not having any striping and move on, rather than returning this error to the caller. Don't crash in cb_getdirstripe() if it is called with a NULL directory pointer or no directory is opened. Lustre-change: https://review.whamcloud.com/43866 Lustre-commit: baba1fd07a977a62295482919e9218f877c0535a Test-Parameters: trivial Signed-off-by: Andreas Dilger Change-Id: If8dd135a86a6a8911bf804542132b2e7a3ce7057 Reviewed-by: Lai Siyao Reviewed-by: Olaf Faaland-LLNL Reviewed-on: https://review.whamcloud.com/44945 Tested-by: jenkins Tested-by: Maloo --- lustre/doc/lfs-find.1 | 8 ++- lustre/tests/sanity.sh | 30 ++++++++ lustre/utils/liblustreapi.c | 164 +++++++++++++++++++++++--------------------- 3 files changed, 123 insertions(+), 79 deletions(-) diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 286b2cd..cd278ea 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -165,9 +165,15 @@ Simple hash function that sums all of the characters in the filename. This is mostly for testing, or if it is known that filenames will use sequential filenames. .RE +This implicitly selects only directories to be matched, like +.B -type d +and not other file types. .TP .BR --mdt-count | -T -The DNE striped directory has the given number of MDT shards. +The DNE striped directory has the given number of MDT shards. This +implicitly selects only directories to be matched, like +.B -type d +and not other file types. .TP .BR --mirror-count | -N The file has \fIn\fR mirrors in its layout. diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index e185564..1354800 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -6442,6 +6442,36 @@ test_56rb() { } run_test 56rb "check lfs find --size --ost/--mdt works" +test_56rc() { + (( MDSCOUNT >= 2 )) || skip "needs at least 2 MDTs" + local dir=$DIR/$tdir + local found + + test_mkdir -c 2 -H all_char $dir || error "failed to mkdir $dir" + $LFS mkdir -c 2 --mdt-hash all_char $dir/$tdir-all{1..10} + (( $MDSCOUNT > 2 )) && + $LFS mkdir -c 3 --mdt-hash fnv_1a_64 $dir/$tdir-fnv{1..10} + mkdir $dir/$tdir-{1..10} + touch $dir/$tfile-{1..10} + + found=$($LFS find $dir --mdt-count 2 | wc -l) + expect=11 + (( $found == $expect )) || error "found $found 2-stripe, expect $expect" + + found=$($LFS find $dir -T +1 | wc -l) + (( $MDSCOUNT > 2 )) && expect=$((expect + 10)) + (( $found == $expect )) || error "found $found 2+stripe, expect $expect" + + found=$($LFS find $dir --mdt-hash all_char | wc -l) + expect=11 + (( $found == $expect )) || error "found $found all_char, expect $expect" + + found=$($LFS find $dir --mdt-hash fnv_1a_64 | wc -l) + (( $MDSCOUNT > 2 )) && expect=10 || expect=0 + (( $found == $expect )) || error "found $found all_char, expect $expect" +} +run_test 56rc "check lfs find --mdt-count/--mdt-hash works" + test_56s() { # LU-611 #LU-9369 [[ $OSTCOUNT -lt 2 ]] && skip_env "need at least 2 OSTs" diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 4b72d8b..ad307da 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -1788,6 +1788,11 @@ typedef int (semantic_func_t)(char *path, DIR *parent, DIR **d, #define OBD_NOT_FOUND (-1) +static bool lmv_is_foreign(__u32 magic) +{ + return magic == LMV_MAGIC_FOREIGN; +} + static void find_param_fini(struct find_param *param) { if (param->fp_migrate) @@ -1884,6 +1889,8 @@ static int cb_get_dirstripe(char *path, DIR *d, struct find_param *param) { int ret; + if (!d) + return -ENOTDIR; again: param->fp_lmv_md->lum_stripe_count = param->fp_lmv_stripe_count; if (param->fp_get_default_lmv) @@ -1897,7 +1904,7 @@ again: int lmv_size; /* if foreign LMV case, fake stripes number */ - if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { + if (lmv_is_foreign(param->fp_lmv_md->lum_magic)) { struct lmv_foreign_md *lfm; lfm = (struct lmv_foreign_md *)param->fp_lmv_md; @@ -1933,6 +1940,7 @@ again: } goto again; } + return ret; } @@ -4566,7 +4574,7 @@ static int find_check_foreign(struct find_param *param) struct lmv_foreign_md *lfm; lfm = (void *)param->fp_lmv_md; - if (lfm->lfm_magic != LMV_MAGIC_FOREIGN) { + if (lmv_is_foreign(lfm->lfm_magic)) { if (param->fp_foreign_type == LU_FOREIGN_TYPE_UNKNOWN) return param->fp_exclude_foreign ? 1 : -1; return -1; @@ -4801,16 +4809,21 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, } /* See if we can check the file type from the dirent. */ - if (param->fp_type != 0 && de != NULL && de->d_type != DT_UNKNOWN) { - checked_type = 1; + if (de != NULL && de->d_type != DT_UNKNOWN) { + if (param->fp_type != 0) { + checked_type = 1; - if (DTTOIF(de->d_type) == param->fp_type) { - if (param->fp_exclude_type) - goto decided; - } else { - if (!param->fp_exclude_type) - goto decided; + if (DTTOIF(de->d_type) == param->fp_type) { + if (param->fp_exclude_type) + goto decided; + } else { + if (!param->fp_exclude_type) + goto decided; + } } + if ((param->fp_check_mdt_count || param->fp_hash_type || + param->fp_check_hash_flag) && de->d_type != DT_DIR) + goto decided; } ret = 0; @@ -4833,23 +4846,66 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, decision = 0; if (decision == 0) { - if (dir && (param->fp_check_mdt_count || - param->fp_hash_type || param->fp_check_foreign || - param->fp_check_hash_flag)) { + if (dir && + (param->fp_check_mdt_count || param->fp_hash_type || + param->fp_check_hash_flag || param->fp_check_foreign)) { param->fp_get_lmv = 1; ret = cb_get_dirstripe(path, dir, param); if (ret != 0) { - /* - * XXX this works to decide for foreign - * criterion only - */ - if (errno == ENODATA && - param->fp_check_foreign) { - if (param->fp_exclude_foreign) - goto foreign; - goto decided; + if (errno == ENODATA) { + ret = 0; + if (param->fp_check_mdt_count || + param->fp_hash_type || + param->fp_check_hash_flag) { + param->fp_lmv_md->lum_stripe_count = 0; + param->fp_lmv_md->lum_hash_type = 0; + } + if (param->fp_check_foreign) { + if (param->fp_exclude_foreign) + goto print; + goto decided; + } + } else { + return ret; } - return ret; + } + + if (param->fp_check_mdt_count) { + if (lmv_is_foreign(param->fp_lmv_md->lum_magic)) + goto decided; + + decision = find_value_cmp(param->fp_lmv_md->lum_stripe_count, + param->fp_mdt_count, + param->fp_mdt_count_sign, + param->fp_exclude_mdt_count, 1, 0); + if (decision == -1) + goto decided; + } + + if (param->fp_hash_type) { + __u32 found; + __u32 type = param->fp_lmv_md->lum_hash_type & + LMV_HASH_TYPE_MASK; + + if (lmv_is_foreign(param->fp_lmv_md->lum_magic)) + goto decided; + + found = (1 << type) & param->fp_hash_type; + if ((found && param->fp_exclude_hash_type) || + (!found && !param->fp_exclude_hash_type)) + goto decided; + } + + if (param->fp_check_hash_flag) { + __u32 flags = param->fp_lmv_md->lum_hash_type & + ~LMV_HASH_TYPE_MASK; + + if (lmv_is_foreign(param->fp_lmv_md->lum_magic)) + goto decided; + + if (!(flags & param->fp_hash_inflags) || + (flags & param->fp_hash_exflags)) + goto decided; } } @@ -4915,6 +4971,10 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, } if (param->fp_type && !checked_type) { + if ((param->fp_check_mdt_count || param->fp_check_hash_flag || + param->fp_hash_type) && !S_ISDIR(lmd->lmd_stx.stx_mode)) + goto decided; + if ((lmd->lmd_stx.stx_mode & S_IFMT) == param->fp_type) { if (param->fp_exclude_type) goto decided; @@ -4978,61 +5038,12 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, goto decided; } - if (param->fp_check_mdt_count) { - if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { - decision = -1; - goto decided; - } - - decision = find_value_cmp( - param->fp_lmv_md->lum_stripe_count, - param->fp_mdt_count, - param->fp_mdt_count_sign, - param->fp_exclude_mdt_count, 1, 0); - if (decision == -1) - goto decided; - } - if (param->fp_check_layout) { decision = find_check_layout(param); if (decision == -1) goto decided; } - if (param->fp_hash_type) { - __u32 found; - __u32 type = param->fp_lmv_md->lum_hash_type & - LMV_HASH_TYPE_MASK; - - if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { - decision = -1; - goto decided; - } - - found = (1 << type) & param->fp_hash_type; - if ((found && param->fp_exclude_hash_type) || - (!found && !param->fp_exclude_hash_type)) { - decision = -1; - goto decided; - } - } - - if (param->fp_check_hash_flag) { - __u32 flags = param->fp_lmv_md->lum_hash_type & - ~LMV_HASH_TYPE_MASK; - - if (param->fp_lmv_md->lum_magic == LMV_MAGIC_FOREIGN) { - decision = -1; - goto decided; - } - - if (!(flags & param->fp_hash_inflags) || - (flags & param->fp_hash_exflags)) { - decision = -1; - goto decided; - } - } - /* If an OBD UUID is specified but none matches, skip this file. */ if ((param->fp_obd_uuid && param->fp_obd_index == OBD_NOT_FOUND) || (param->fp_mdt_uuid && param->fp_mdt_index == OBD_NOT_FOUND)) @@ -5259,12 +5270,9 @@ obd_matches: goto decided; } -foreign: - llapi_printf(LLAPI_MSG_NORMAL, "%s", path); - if (param->fp_zero_end) - llapi_printf(LLAPI_MSG_NORMAL, "%c", '\0'); - else - llapi_printf(LLAPI_MSG_NORMAL, "\n"); +print: + llapi_printf(LLAPI_MSG_NORMAL, "%s%c", path, + param->fp_zero_end ? '\0' : '\n'); decided: ret = 0; -- 1.8.3.1