From c99a393125bce54f9e673e102f8d65aca38f8114 Mon Sep 17 00:00:00 2001 From: Yang Sheng Date: Mon, 10 Apr 2017 15:31:26 +0800 Subject: [PATCH] LU-7502 utils: add --mdt-count & --mdt-hash options lfs find --mdt-count and --mdt-hash options lfs getdirstripe --mdt-count and --mdt-hash options Signed-off-by: Yang Sheng Change-Id: I55f2d7660c4e4fea5a197a57407b614c4004974c Reviewed-on: https://review.whamcloud.com/25515 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Bob Glossman Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lustre/include/lustre/lustreapi.h | 11 ++- lustre/tests/sanity.sh | 26 +++++- lustre/utils/lfs.c | 192 +++++++++++++++++++++++++------------- lustre/utils/liblustreapi.c | 30 +++++- 4 files changed, 186 insertions(+), 73 deletions(-) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 886409d..6b13669 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -175,7 +175,8 @@ struct find_param { fp_stripe_count_sign:2, fp_comp_start_sign:2, fp_comp_end_sign:2, - fp_comp_count_sign:2; + fp_comp_count_sign:2, + fp_mdt_count_sign:2; unsigned long long fp_size; unsigned long long fp_size_units; @@ -215,7 +216,11 @@ struct find_param { fp_check_comp_end:1, fp_exclude_comp_end:1, fp_check_comp_id:1, - fp_exclude_comp_id:1; + fp_exclude_comp_id:1, + fp_check_mdt_count:1, + fp_exclude_mdt_count:1, + fp_check_hash_type:1, + fp_exclude_hash_type:1; int fp_verbose; int fp_quiet; @@ -256,11 +261,13 @@ struct find_param { unsigned long long fp_comp_start_units; unsigned long long fp_comp_end; unsigned long long fp_comp_end_units; + unsigned long long fp_mdt_count; /* In-process parameters. */ unsigned long fp_got_uuids:1, fp_obds_printed:1; unsigned int fp_depth; + unsigned int fp_hash_type; }; extern int llapi_ostlist(char *path, struct find_param *param); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 33268f1..701aeeb 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -14729,7 +14729,11 @@ test_striped_dir() { error "getdirstripe failed" stripe_count=$($LFS getdirstripe -c $DIR/$tdir/striped_dir) if [ "$stripe_count" != "2" ]; then - error "stripe_count is $stripe_count, expect 2" + error "1:stripe_count is $stripe_count, expect 2" + fi + stripe_count=$($LFS getdirstripe -T $DIR/$tdir/striped_dir) + if [ "$stripe_count" != "2" ]; then + error "2:stripe_count is $stripe_count, expect 2" fi stripe_index=$($LFS getdirstripe -i $DIR/$tdir/striped_dir) @@ -14895,13 +14899,13 @@ test_300e() { mkdir $DIR/$tdir/striped_dir/dir_c $LFS setdirstripe -i 0 -c 2 -t all_char $DIR/$tdir/striped_dir/stp_a || - error "set striped dir under striped dir error" + error "set striped adir under striped dir error" - $LFS setdirstripe -i 0 -c 2 -t all_char $DIR/$tdir/striped_dir/stp_b || - error "set striped dir under striped dir error" + $LFS setdirstripe -i 0 -c 2 -H all_char $DIR/$tdir/striped_dir/stp_b || + error "set striped bdir under striped dir error" $LFS setdirstripe -i 0 -c 2 -t all_char $DIR/$tdir/striped_dir/stp_c || - error "set striped dir under striped dir error" + error "set striped cdir under striped dir error" mrename $DIR/$tdir/striped_dir/dir_a $DIR/$tdir/striped_dir/dir_b || error "rename dir under striped dir fails" @@ -15105,11 +15109,23 @@ test_300i() { createmany -o $DIR/$tdir/striped_dir/f- 10 || error "create files under striped dir failed" + $LFS setdirstripe -i0 -c$MDSCOUNT -H all_char $DIR/$tdir/hashdir || + error "set striped hashdir error" + + $LFS setdirstripe -i0 -c$MDSCOUNT -H all_char $DIR/$tdir/hashdir/d0 || + error "create dir0 under hash dir failed" + $LFS setdirstripe -i0 -c$MDSCOUNT -H fnv_1a_64 $DIR/$tdir/hashdir/d1 || + error "create dir1 under hash dir failed" + # unfortunately, we need to umount to clear dir layout cache for now # once we fully implement dir layout, we can drop this umount_client $MOUNT || error "umount failed" mount_client $MOUNT || error "mount failed" + $LFS find -H fnv_1a_64 $DIR/$tdir/hashdir + local dircnt=$($LFS find -H fnv_1a_64 $DIR/$tdir/hashdir | wc -l) + [ $dircnt -eq 1 ] || error "lfs find striped dir got:$dircnt,except:1" + #set the stripe to be unknown hash type #define OBD_FAIL_UNKNOWN_LMV_STRIPE 0x1901 $LCTL set_param fail_loc=0x1901 diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index e0eadb5..89f9597 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -157,7 +157,7 @@ static int lfs_list_commands(int argc, char **argv); #define SETDIRSTRIPE_USAGE \ " [--mdt-count|-c stripe_count>\n" \ " [--mdt-index|-i mdt_index]\n" \ - " [--mdt-hash|-t mdt_hash]\n" \ + " [--mdt-hash|-H mdt_hash]\n" \ " [--default|-D] [--mode|-m mode] \n" \ "\tstripe_count: stripe count of the striped directory\n" \ "\tmdt_index: MDT index of first stripe\n" \ @@ -217,7 +217,7 @@ command_t cmdlist[] = { "To list the striping info for a given directory\n" "or recursively for all directories in a directory tree.\n" "usage: getdirstripe [--obd|-O ] [--mdt-count|-c]\n" - " [--mdt-index|-i] [--mdt-hash|-t]\n" + " [--mdt-index|-i] [--mdt-hash|-H]\n" " [--recursive|-r] [--default|-D] ..."}, {"mkdir", lfs_setdirstripe, 0, "To create a striped directory on a specified MDT. This can only\n" @@ -251,9 +251,14 @@ command_t cmdlist[] = { " [[!] --component-start [+-]N[kMGTPE]]\n" " [[!] --component-end|-E [+-]N[kMGTPE]]\n" " [[!] --component-flags ]\n" + " [[!] --mdt-count|-T [+-]]\n" + " [[!] --mdt-hash|-H \n" "\t !: used before an option indicates 'NOT' requested attribute\n" "\t -: used before a value indicates 'AT MOST' requested value\n" - "\t +: used before a value indicates 'AT LEAST' requested value\n"}, + "\t +: used before a value indicates 'AT LEAST' requested value\n" + "\tmdt-hash: hash type of the striped directory.\n" + "\t fnv_1a_64 FNV-1a hash algorithm\n" + "\t all_char sum of characters % MDT_COUNT\n"}, {"check", lfs_check, 0, "Display the status of MDS or OSTs (as specified in the command)\n" "or all the servers (MDS and OSTs).\n" @@ -408,6 +413,17 @@ command_t cmdlist[] = { #define MIGRATION_NONBLOCK 1 +static int check_hashtype(const char *hashtype) +{ + int i; + + for (i = LMV_HASH_TYPE_ALL_CHARS; i < LMV_HASH_TYPE_MAX; i++) + if (strcmp(hashtype, mdt_hash_name[i]) == 0) + return i; + + return 0; +} + /** * Internal helper for migrate_copy_data(). Check lease and report error if * need be. @@ -1306,6 +1322,7 @@ static int lfs_setstripe(int argc, char **argv) {"stripe_count", required_argument, 0, 'c'}, {"delete", no_argument, 0, 'd'}, {"component-end", required_argument, 0, 'E'}, + /* dirstripe {"mdt-hash", required_argument, 0, 'H'}, */ #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) /* This formerly implied "stripe-index", but was explicitly * made "stripe-index" for consistency with other options, @@ -1334,6 +1351,7 @@ static int lfs_setstripe(int argc, char **argv) #endif {"stripe-size", required_argument, 0, 'S'}, {"stripe_size", required_argument, 0, 'S'}, + /* dirstripe {"mdt-count", required_argument, 0, 'T'}, */ /* --verbose is only valid in migrate mode */ {"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0} @@ -1811,6 +1829,7 @@ static int lfs_find(int argc, char **argv) {"component-end", required_argument, 0, 'E'}, {"gid", required_argument, 0, 'g'}, {"group", required_argument, 0, 'G'}, + {"mdt-hash", required_argument, 0, 'H'}, {"stripe-index", required_argument, 0, 'i'}, {"stripe_index", required_argument, 0, 'i'}, /*{"component-id", required_argument, 0, 'I'},*/ @@ -1831,6 +1850,7 @@ static int lfs_find(int argc, char **argv) {"stripe-size", required_argument, 0, 'S'}, {"stripe_size", required_argument, 0, 'S'}, {"type", required_argument, 0, 't'}, + {"mdt-count", required_argument, 0, 'T'}, {"uid", required_argument, 0, 'u'}, {"user", required_argument, 0, 'U'}, {0, 0, 0, 0} @@ -1847,8 +1867,8 @@ static int lfs_find(int argc, char **argv) /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */ while ((c = getopt_long_only(argc, argv, - "-A:c:C:D:E:g:G:i:L:m:M:n:O:Ppqrs:S:t:u:U:v", - long_opts, NULL)) >= 0) { + "-A:c:C:D:E:g:G:H:i:L:m:M:n:O:Ppqrs:S:t:T:u:U:v", + long_opts, NULL)) >= 0) { xtime = NULL; xsign = NULL; if (neg_opt) @@ -2012,6 +2032,17 @@ static int lfs_find(int argc, char **argv) param.fp_exclude_gid = !!neg_opt; param.fp_check_gid = 1; break; + case 'H': + param.fp_hash_type = check_hashtype(optarg); + if (param.fp_hash_type == 0) { + fprintf(stderr, "error: bad hash_type '%s'\n", + optarg); + ret = -1; + goto err; + } + param.fp_check_hash_type = 1; + param.fp_exclude_hash_type = !!neg_opt; + break; case 'L': ret = name2layout(¶m.fp_layout, optarg); if (ret) @@ -2205,6 +2236,25 @@ err_free: goto err; }; break; + case 'T': + if (optarg[0] == '+') { + param.fp_mdt_count_sign = -1; + optarg++; + } else if (optarg[0] == '-') { + param.fp_mdt_count_sign = 1; + optarg++; + } + + param.fp_mdt_count = strtoul(optarg, &endptr, 0); + if (*endptr != '\0') { + fprintf(stderr, "error: bad mdt_count '%s'\n", + optarg); + ret = -1; + goto err; + } + param.fp_check_mdt_count = 1; + param.fp_exclude_mdt_count = !!neg_opt; + break; default: ret = CMD_HELP; goto err; @@ -2260,6 +2310,7 @@ static int lfs_getstripe_internal(int argc, char **argv, {"component-end", required_argument, 0, 'E'}, {"fid", no_argument, 0, 'F'}, {"generation", no_argument, 0, 'g'}, + /* dirstripe {"mdt-hash", required_argument, 0, 'H'}, */ #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) /* This formerly implied "stripe-index", but was explicitly * made "stripe-index" for consistency with other options, @@ -2297,6 +2348,7 @@ static int lfs_getstripe_internal(int argc, char **argv, #endif {"stripe-size", no_argument, 0, 'S'}, {"stripe_size", no_argument, 0, 'S'}, + /* dirstripe {"mdt-count", required_argument, 0, 'T'}, */ {"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0} }; @@ -2306,17 +2358,14 @@ static int lfs_getstripe_internal(int argc, char **argv, while ((c = getopt_long(argc, argv, "cdDE:FghiI:LmMoO:pqrRsSv", long_opts, NULL)) != -1) { switch (c) { - case 'O': - if (param->fp_obd_uuid) { - fprintf(stderr, - "error: %s: only one obduuid allowed", - argv[0]); - return CMD_HELP; + case 'c': + if (strcmp(argv[optind - 1], "--count") == 0) + fprintf(stderr, "warning: '--count' deprecated," + " use '--stripe-count' instead\n"); + if (!(param->fp_verbose & VERBOSE_DETAIL)) { + param->fp_verbose |= VERBOSE_COUNT; + param->fp_max_depth = 0; } - param->fp_obd_uuid = (struct obd_uuid *)optarg; - break; - case 'q': - param->fp_quiet++; break; case LFS_COMP_COUNT_OPT: param->fp_verbose |= VERBOSE_COMP_COUNT; @@ -2401,33 +2450,9 @@ static int lfs_getstripe_internal(int argc, char **argv, param->fp_max_depth = 0; } break; - case 'r': - param->fp_recursive = 1; - break; - case 'v': - param->fp_verbose = VERBOSE_DEFAULT | VERBOSE_DETAIL; - break; - case 'c': -#if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 6, 53, 0) - if (strcmp(argv[optind - 1], "--count") == 0) - fprintf(stderr, "warning: '--count' deprecated," - " use '--stripe-count' instead\n"); -#endif - if (!(param->fp_verbose & VERBOSE_DETAIL)) { - param->fp_verbose |= VERBOSE_COUNT; - param->fp_max_depth = 0; - } - break; -#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) - case 's': -#if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 6, 53, 0) - fprintf(stderr, "warning: '--size|-s' deprecated, " - "use '--stripe-size|-S' instead\n"); -#endif -#endif /* LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) */ - case 'S': + case 'g': if (!(param->fp_verbose & VERBOSE_DETAIL)) { - param->fp_verbose |= VERBOSE_SIZE; + param->fp_verbose |= VERBOSE_GENERATION; param->fp_max_depth = 0; } break; @@ -2462,18 +2487,6 @@ static int lfs_getstripe_internal(int argc, char **argv, param->fp_verbose |= VERBOSE_COMP_ID; } break; - case 'p': - if (!(param->fp_verbose & VERBOSE_DETAIL)) { - param->fp_verbose |= VERBOSE_POOL; - param->fp_max_depth = 0; - } - break; - case 'g': - if (!(param->fp_verbose & VERBOSE_DETAIL)) { - param->fp_verbose |= VERBOSE_GENERATION; - param->fp_max_depth = 0; - } - break; case 'L': if (!(param->fp_verbose & VERBOSE_DETAIL)) { param->fp_verbose |= VERBOSE_LAYOUT; @@ -2492,9 +2505,44 @@ static int lfs_getstripe_internal(int argc, char **argv, param->fp_max_depth = 0; param->fp_verbose |= VERBOSE_MDTINDEX; break; + case 'O': + if (param->fp_obd_uuid) { + fprintf(stderr, + "error: %s: only one obduuid allowed", + argv[0]); + return CMD_HELP; + } + param->fp_obd_uuid = (struct obd_uuid *)optarg; + break; + case 'p': + if (!(param->fp_verbose & VERBOSE_DETAIL)) { + param->fp_verbose |= VERBOSE_POOL; + param->fp_max_depth = 0; + } + break; + case 'q': + param->fp_quiet++; + break; + case 'r': + param->fp_recursive = 1; + break; case 'R': param->fp_raw = 1; break; +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) + case 's': + fprintf(stderr, "warning: '--size|-s' deprecated, " + "use '--stripe-size|-S' instead\n"); +#endif /* LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 59, 0) */ + case 'S': + if (!(param->fp_verbose & VERBOSE_DETAIL)) { + param->fp_verbose |= VERBOSE_SIZE; + param->fp_max_depth = 0; + } + break; + case 'v': + param->fp_verbose = VERBOSE_DEFAULT | VERBOSE_DETAIL; + break; default: return CMD_HELP; } @@ -2574,19 +2622,26 @@ static int lfs_getdirstripe(int argc, char **argv) { struct find_param param = { 0 }; struct option long_opts[] = { +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) {"mdt-count", no_argument, 0, 'c'}, +#endif + {"mdt-hash", no_argument, 0, 'H'}, {"mdt-index", no_argument, 0, 'i'}, {"recursive", no_argument, 0, 'r'}, +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) {"mdt-hash", no_argument, 0, 't'}, +#endif {"default", no_argument, 0, 'D'}, {"obd", required_argument, 0, 'O'}, + {"mdt-count", no_argument, 0, 'T'}, {0, 0, 0, 0} }; int c, rc; param.fp_get_lmv = 1; - while ((c = getopt_long(argc, argv, "cirtDO:", long_opts, NULL)) != -1) + while ((c = getopt_long(argc, argv, + "cDHiO:rtT", long_opts, NULL)) != -1) { switch (c) { case 'O': @@ -2598,13 +2653,23 @@ static int lfs_getdirstripe(int argc, char **argv) } param.fp_obd_uuid = (struct obd_uuid *)optarg; break; +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) case 'c': +#if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 10, 50, 0) + fprintf(stderr, "warning: '-c' deprecated" + ", use '-T' instead\n"); +#endif +#endif + case 'T': param.fp_verbose |= VERBOSE_COUNT; break; case 'i': param.fp_verbose |= VERBOSE_OFFSET; break; +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) case 't': +#endif + case 'H': param.fp_verbose |= VERBOSE_HASH_TYPE; break; case 'D': @@ -2669,8 +2734,9 @@ static int lfs_setdirstripe(int argc, char **argv) {"mode", required_argument, 0, 'm'}, #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) {"hash-type", required_argument, 0, 't'}, -#endif {"mdt-hash", required_argument, 0, 't'}, +#endif + {"mdt-hash", required_argument, 0, 'H'}, #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) {"default_stripe", no_argument, 0, 'D'}, #endif @@ -2678,7 +2744,7 @@ static int lfs_setdirstripe(int argc, char **argv) {0, 0, 0, 0} }; - while ((c = getopt_long(argc, argv, "c:dDi:m:t:", long_opts, + while ((c = getopt_long(argc, argv, "c:dDi:H:m:t:", long_opts, NULL)) >= 0) { switch (c) { case 0: @@ -2710,7 +2776,10 @@ static int lfs_setdirstripe(int argc, char **argv) case 'm': mode_opt = optarg; break; +#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) case 't': +#endif + case 'H': #if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 11, 53, 0) if (strcmp(argv[optind - 1], "--hash-type") == 0) fprintf(stderr, "warning: '--hash-type' " @@ -2773,20 +2842,13 @@ static int lfs_setdirstripe(int argc, char **argv) if (stripe_hash_opt == NULL) { hash_type = LMV_HASH_TYPE_FNV_1A_64; } else { - int i; - - for (i = LMV_HASH_TYPE_ALL_CHARS; i < LMV_HASH_TYPE_MAX; i++) - if (strcmp(stripe_hash_opt, mdt_hash_name[i]) == 0) - break; - - if (i == LMV_HASH_TYPE_MAX) { + hash_type = check_hashtype(stripe_hash_opt); + if (hash_type == 0) { fprintf(stderr, "error: %s: bad stripe hash type '%s'\n", argv[0], stripe_hash_opt); return CMD_HELP; } - - hash_type = i; } /* get the stripe count */ diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 022dad7..2b2bae8 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -3321,13 +3321,20 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, if (param->fp_obd_uuid || param->fp_mdt_uuid || param->fp_check_uid || param->fp_check_gid || param->fp_atime || param->fp_mtime || param->fp_ctime || - param->fp_check_size || find_check_lmm_info(param)) + param->fp_check_size || find_check_lmm_info(param) || + param->fp_check_mdt_count || param->fp_check_hash_type) decision = 0; if (param->fp_type != 0 && checked_type == 0) decision = 0; if (decision == 0) { + if (param->fp_check_mdt_count || param->fp_check_hash_type) { + param->fp_get_lmv = 1; + ret = cb_get_dirstripe(path, dir, param); + if (ret != 0) + return ret; + } ret = get_lmd_info(path, parent, dir, param->fp_lmd, param->fp_lum_size); if (ret == 0 && param->fp_lmd->lmd_lmm.lmm_magic == 0 && @@ -3435,12 +3442,33 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, goto decided; } + if (param->fp_check_mdt_count) { + decision = find_value_cmp( + param->fp_lmv_md->lum_stripe_count, + param->fp_mdt_count, + param->fp_mdt_count_sign, + param->fp_exclude_mdt_count, 1, 0); + if (decision == -1) + goto decided; + } + if (param->fp_check_layout) { decision = find_check_layout(param); if (decision == -1) goto decided; } + if (param->fp_check_hash_type) { + __u32 found; + + found = param->fp_lmv_md->lum_hash_type & param->fp_hash_type; + if ((found && param->fp_exclude_hash_type) || + (!found && !param->fp_exclude_hash_type)) { + decision = -1; + goto decided; + } + } + /* If an OBD UUID is specified but none matches, skip this file. */ if ((param->fp_obd_uuid && param->fp_obd_index == OBD_NOT_FOUND) || (param->fp_mdt_uuid && param->fp_mdt_index == OBD_NOT_FOUND)) -- 1.8.3.1