From 45c557ece5186800f5567df7e1c58909bb25b338 Mon Sep 17 00:00:00 2001 From: Frederick Dilger Date: Thu, 6 Jun 2024 16:00:25 -0400 Subject: [PATCH] LU-16491 utils: update getdirstripe yaml format 'lfs getdirstripe --yaml' now prints directory layout in yaml format. getdirstripe now also prints the "self" FID whether the directory is striped or not. "migrating" fields (lmv_migrate_offset, lmv_migrate_hash) were not included because of the additional code complexity required to add the two fields. The migrating fields are stored in 'struct lmv_mds_md_v1' which AFAIK isn't available though getdirstripe. For 0 striped directories, lmv_objects: will now contain information on the directory itself, this information becomes redundant with -v, however it is useful when the lmv_fid isn't being shown. New YAML layout: lmv_fid: 0x280000404:0x5:0x0 lmv_magic: 0xcd20cd0 lmv_stripe_count: 4 lmv_stripe_offset: 2 lmv_hash_type: crush lmv_objects: - l_mdt_idx: 2 l_fid: 0x280000400:0x4:0x0 - l_mdt_idx: 0 l_fid: 0x200000402:0x4:0x0 - l_mdt_idx: 1 l_fid: 0x240000402:0x2:0x0 - l_mdt_idx: 3 l_fid: 0x2c0000403:0x2:0x0 Signed-off-by: Frederick Dilger Change-Id: I03ddc24816484d11c8c70892831e9edc9da5455a Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55346 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Lai Siyao Reviewed-by: Oleg Drokin --- lustre/doc/lfs-getdirstripe.1 | 3 ++ lustre/tests/sanity.sh | 32 +++++++++++- lustre/tests/test-framework.sh | 29 ++++++++++- lustre/utils/lfs.c | 19 ++++--- lustre/utils/liblustreapi.c | 111 +++++++++++++++++++++++++---------------- 5 files changed, 142 insertions(+), 52 deletions(-) diff --git a/lustre/doc/lfs-getdirstripe.1 b/lustre/doc/lfs-getdirstripe.1 index 7f98a88..ea2e7e2 100644 --- a/lustre/doc/lfs-getdirstripe.1 +++ b/lustre/doc/lfs-getdirstripe.1 @@ -17,6 +17,9 @@ Only show the number of MDTs the directory is striped across. .BR \-D ", " \-\-default Show the default layout used when creating new subdirectories. .TP +.BR \-F ", " \-\-fid +Show only the 128-bit unique Lustre File Identifier (FID). +.TP .BR \-h ", " \-\-help Print usage message. .TP diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 9ae3722..b6df61d 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -6662,10 +6662,40 @@ test_56bb() { $LFS getdirstripe -v -D -y $DIR 1> $output_file cat $output_file - cat $output_file | verify_yaml || error "layout is not valid YAML" + verify_yaml <<<$output_file || + error "default layout is not valid YAML" + +# check non-default 1 stripe case + test_mkdir -c 1 $DIR/$tdir + $LFS getdirstripe -v -y $DIR 1> $output_file + cat $output_file + verify_yaml <<<$output_file || error "dir layout not valid YAML" } run_test 56bb "check $LFS getdirstripe layout is YAML" +test_56bc() { + (( $MDSCOUNT >= 2 )) || skip "need >= 2 MDTs" + verify_yaml_available || skip_env "YAML verification not installed" + + local src_dir=$DIR/$tdir.src + local yaml_file=$DIR/$tfile.yaml + + $LFS setdirstripe -c 4 $src_dir || + error "failed to setstripe" + $LFS getdirstripe --yaml -v $src_dir > $yaml_file || + error "failed to getdirstripe --yaml -v" + + cat $yaml_file + verify_yaml <<<$yaml_file || + error "striped dir layout is not valid YAML" + + local orig=$(get_dir_layout_param $src_dir) + local rest=$(cat $yaml_file | parse_layout_param) + [[ "$orig" == "$rest" ]] || + error "failed to parse YAML layout" +} +run_test 56bc "check '$LFS getdirstripe --yaml' params are valid" + test_56c() { remote_ost_nodsh && skip "remote OST with nodsh" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 69b598f..94ed56f 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -10584,6 +10584,22 @@ parse_plain_param() fi } +parse_dir_param() +{ + local line=$1 + local val=$(awk '{print $2}' <<< $line) + + if [[ $line =~ ^"lmv_stripe_count:" ]]; then + echo "-c $val" + elif [[ $line =~ ^"lmv_stripe_offset:" ]]; then + echo "-i $val" + elif [[ $line =~ ^"lmv_hash_type:" ]]; then + echo "-H $val" + elif [[ $line =~ ^"lmv_max_inherit:" ]]; then + echo "-X $val" + fi +} + parse_layout_param() { local mode="" @@ -10599,6 +10615,8 @@ parse_layout_param() mode="plain_file" elif [[ $line =~ ^"lcm_layout_gen:" ]]; then mode="pfl" + elif [[ $line =~ ^"lmv_stripe_count" ]]; then + mode="dne" fi fi @@ -10624,6 +10642,9 @@ parse_layout_param() val=$(parse_plain_param "$line") [[ ! -z $val ]] && param="$param $val" fi + elif [[ $mode = "dne" ]]; then + val=$(parse_dir_param "$line") + [[ ! -z $val ]] && param="$param $val" fi fi done @@ -10632,7 +10653,13 @@ parse_layout_param() get_layout_param() { - local param=$($LFS getstripe -d $1 | parse_layout_param) + local param=$($LFS getstripe -dy $1 | parse_layout_param) + echo "$param" +} + +get_dir_layout_param() +{ + local param=$($LFS getdirstripe -y $1 | parse_layout_param) echo "$param" } diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index c83c6fc..cf38041 100755 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -352,12 +352,12 @@ command_t cmdlist[] = { {"getdirstripe", lfs_getdirstripe, 0, "To list the layout pattern info for a given directory\n" "or recursively for all directories in a directory tree.\n" - "usage: getdirstripe [--mdt-count|-c] [--mdt-index|-m|-i]\n" + "usage: getdirstripe [--mdt-count|-c] [--default|-D] [--fid|-F]\n" " [--help|-h] [--hex-idx] [--mdt-hash|-H]\n" - " [--obd|-O UUID] [--recursive|-r] [--raw|-R]\n" - " [--yaml|-y] [--verbose|-v] [--default|-D]\n" - " [--max-inherit|-X]\n" - " [--max-inherit-rr] ..."}, + " [--mdt-index|-m|-i] [--obd|-O UUID]\n" + " [--recursive|-r] [--raw|-R]\n" + " [--verbose|-v] [--max-inherit|-X]\n" + " [--max-inherit-rr] [--yaml|-y] ...\n"}, {"mkdir", lfs_setdirstripe, 0, "Create striped directory on specified MDT, same as setdirstripe.\n" "usage: mkdir [OPTION] \n" @@ -6778,6 +6778,7 @@ static int lfs_getdirstripe(int argc, char **argv) struct option long_opts[] = { { .val = 'c', .name = "mdt-count", .has_arg = no_argument }, { .val = 'D', .name = "default", .has_arg = no_argument }, + { .val = 'F', .name = "fid", .has_arg = no_argument }, { .val = 'h', .name = "help", .has_arg = no_argument }, { .val = 'H', .name = "mdt-hash", .has_arg = no_argument }, { .val = LFS_HEX_IDX_OPT, @@ -6799,7 +6800,7 @@ static int lfs_getdirstripe(int argc, char **argv) param.fp_get_lmv = 1; while ((c = getopt_long(argc, argv, - "cDhHimO:rRtTvXy", long_opts, NULL)) != -1) { + "cDFhHimO:rRtTvXy", long_opts, NULL)) != -1) { switch (c) { case 'c': case 'T': @@ -6808,6 +6809,9 @@ static int lfs_getdirstripe(int argc, char **argv) case 'D': param.fp_get_default_lmv = 1; break; + case 'F': + param.fp_verbose |= VERBOSE_DFID; + break; #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) case 't': fprintf(stderr, @@ -6841,8 +6845,7 @@ static int lfs_getdirstripe(int argc, char **argv) param.fp_raw = 1; break; case 'v': - param.fp_verbose |= VERBOSE_DEFAULT; - param.fp_verbose |= VERBOSE_DETAIL; + param.fp_verbose |= VERBOSE_DEFAULT | VERBOSE_DETAIL; break; case 'X': param.fp_verbose |= VERBOSE_INHERIT; diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 7cb4d63..5231561 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -3060,6 +3060,8 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, bool yaml = flags & LDF_YAML; bool hex = flags & LDF_HEX_IDX; bool obdstripe = false; + struct lu_fid dir_fid; + int rc; int i; if (obdindex != OBD_NOT_FOUND) { @@ -3093,19 +3095,41 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, verbose = VERBOSE_OBJID; } - if (depth && path && ((verbose != VERBOSE_OBJID))) + if (verbose & (VERBOSE_DETAIL | VERBOSE_DFID) || + (verbose & VERBOSE_OBJID && lum->lum_stripe_count >= 0)) { + rc = llapi_path2fid(path, &dir_fid); + if (rc) + llapi_error(LLAPI_MSG_ERROR, rc, + "Cannot determine directory FID: %s", path); + } + + if (depth && path && (verbose != VERBOSE_OBJID)) llapi_printf(LLAPI_MSG_NORMAL, "%s%s\n", prefix, path); + if (verbose & (VERBOSE_DETAIL | VERBOSE_DFID)) { + if (verbose & ~VERBOSE_DFID) + llapi_printf(LLAPI_MSG_NORMAL, + "lmv_fid: %s", yaml ? " " : ""); + llapi_printf(LLAPI_MSG_NORMAL, DFID_NOBRACE, PFID(&dir_fid)); + + separator = yaml ? "\n" : " "; + } + + if (verbose & VERBOSE_DETAIL) { + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); + llapi_printf(LLAPI_MSG_NORMAL, "lmv_magic: %s%#x", + yaml ? " " : "", (int)lum->lum_magic); + separator = yaml ? "\n" : " "; + } + if (verbose & VERBOSE_STRIPE_COUNT) { llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); if (verbose & ~VERBOSE_STRIPE_COUNT) - llapi_printf(LLAPI_MSG_NORMAL, "lmv_stripe_count: "); + llapi_printf(LLAPI_MSG_NORMAL, "lmv_stripe_count: %s", + yaml ? " " : ""); llapi_printf(LLAPI_MSG_NORMAL, "%d", (int)lum->lum_stripe_count); - if ((verbose & VERBOSE_STRIPE_OFFSET) && !yaml) - separator = " "; - else - separator = "\n"; + separator = yaml ? "\n" : " "; } if (verbose & VERBOSE_STRIPE_OFFSET) { @@ -3114,10 +3138,7 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, llapi_printf(LLAPI_MSG_NORMAL, "lmv_stripe_offset: "); llapi_printf(LLAPI_MSG_NORMAL, hex ? "%#x" : "%d", (int)lum->lum_stripe_offset); - if (verbose & VERBOSE_HASH_TYPE && !yaml) - separator = " "; - else - separator = "\n"; + separator = yaml ? "\n" : " "; } if (verbose & VERBOSE_HASH_TYPE) { @@ -3126,7 +3147,8 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); if (verbose & ~VERBOSE_HASH_TYPE) - llapi_printf(LLAPI_MSG_NORMAL, "lmv_hash_type: "); + llapi_printf(LLAPI_MSG_NORMAL, "lmv_hash_type: %s", + yaml ? " " : ""); if (type < LMV_HASH_TYPE_MAX) llapi_printf(LLAPI_MSG_NORMAL, "%s", mdt_hash_name[type]); @@ -3146,17 +3168,14 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, if (flags & ~LMV_HASH_FLAG_KNOWN) llapi_printf(LLAPI_MSG_NORMAL, ",unknown_%04x", flags & ~LMV_HASH_FLAG_KNOWN); - - if (verbose & VERBOSE_HASH_TYPE && !yaml) - separator = " "; - else - separator = "\n"; + separator = yaml ? "\n" : " "; } if ((verbose & VERBOSE_INHERIT) && lum->lum_magic == LMV_USER_MAGIC) { llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); if (verbose & ~VERBOSE_INHERIT) - llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit: "); + llapi_printf(LLAPI_MSG_NORMAL, "lmv_max_inherit: %s", + yaml ? " " : ""); if (lum->lum_max_inherit == LMV_INHERIT_UNLIMITED) llapi_printf(LLAPI_MSG_NORMAL, "-1"); else if (lum->lum_max_inherit == LMV_INHERIT_NONE) @@ -3164,10 +3183,7 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, else llapi_printf(LLAPI_MSG_NORMAL, "%hhu", lum->lum_max_inherit); - if (verbose & VERBOSE_INHERIT && !yaml) - separator = " "; - else - separator = "\n"; + separator = yaml ? "\n" : " "; } if ((verbose & VERBOSE_INHERIT_RR) && @@ -3182,44 +3198,55 @@ static void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, else llapi_printf(LLAPI_MSG_NORMAL, "%hhu", lum->lum_max_inherit_rr); - if (verbose & VERBOSE_INHERIT_RR && !yaml) - separator = " "; - else - separator = "\n"; + separator = yaml ? "\n" : " "; + } + + if ((verbose & VERBOSE_POOL) && pool_name != NULL && + pool_name[0] != '\0') { + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); + if (verbose & ~VERBOSE_POOL) + llapi_printf(LLAPI_MSG_NORMAL, "%slmv_pool: %s", + prefix, yaml ? " " : ""); + llapi_printf(LLAPI_MSG_NORMAL, "%s%c ", pool_name, ' '); } separator = "\n"; if ((verbose & VERBOSE_OBJID) && lum->lum_magic != LMV_USER_MAGIC) { + char fmt[64]; + llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); - if (lum->lum_stripe_count > 0) + if (yaml) + llapi_printf(LLAPI_MSG_NORMAL, + "lmv_objects:\n"); + else if (lum->lum_stripe_count >= 0) llapi_printf(LLAPI_MSG_NORMAL, "mdtidx\t\t FID[seq:oid:ver]\n"); - char fmt[48] = { 0 }; - sprintf(fmt, "%s%s", hex ? "%#6x" : "%6u", - "\t\t "DFID"\t\t%s\n"); + if (yaml) + snprintf(fmt, sizeof(fmt), + " - l_mdt_idx: %s\n%s\n", + hex ? "%#x" : "%d", + " l_fid: "DFID_NOBRACE); + else + snprintf(fmt, sizeof(fmt), "%s%s", hex ? "%#6x" : "%6u", + "\t\t "DFID"\t\t%s\n"); + if (lum->lum_stripe_count == 0 && yaml) { + llapi_printf(LLAPI_MSG_NORMAL, fmt, + lum->lum_stripe_offset, + PFID(&dir_fid), ""); + } for (i = 0; i < lum->lum_stripe_count; i++) { int idx = objects[i].lum_mds; struct lu_fid *fid = &objects[i].lum_fid; if ((obdindex == OBD_NOT_FOUND) || (obdindex == idx)) - llapi_printf(LLAPI_MSG_NORMAL, fmt, - idx, PFID(fid), - obdindex == idx ? " *" : ""); + llapi_printf(LLAPI_MSG_NORMAL, fmt, idx, + PFID(fid), + obdindex == idx ? " *":""); } } - if ((verbose & VERBOSE_POOL) && pool_name != NULL && - pool_name[0] != '\0') { - llapi_printf(LLAPI_MSG_NORMAL, "%s", separator); - if (verbose & ~VERBOSE_POOL) - llapi_printf(LLAPI_MSG_NORMAL, "%slmv_pool: ", - prefix); - llapi_printf(LLAPI_MSG_NORMAL, "%s%c ", pool_name, ' '); - separator = "\n"; - } - if (!(verbose & VERBOSE_OBJID) || lum->lum_magic == LMV_USER_MAGIC) llapi_printf(LLAPI_MSG_NORMAL, "\n"); } -- 1.8.3.1