From 093bd2f3439848d6097e182e5302bf113560d9f9 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Mon, 6 Feb 2023 18:36:52 +0800 Subject: [PATCH] EX-6856 utils: add 'lfs find' support for compressed file * Add "--comp-flags=[^]compress" to locate file with/without compressed components. * Add "--comp-flags=[^]nocompr" to locate file with/without setting component compress preference. * Add "[!] --layout=compress" to locate file with/without compressed components. * Add "[!] --compress-type=" to locate compressed file with/without specified compress algorithm. * Add "[!] --compress-level=[+-]" to locate compressed file with/without specified compress level. * Add "[!] --compress-chunk=[+-]" to locate compressed file with/without specified compress chunk in KiB. Signed-off-by: Bobi Jam Change-Id: Ic36946738c3463fd862aeca4ee2e2c2ed85eff84 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49930 Tested-by: jenkins Tested-by: Andreas Dilger Reviewed-by: Alex Zhuravlev Reviewed-by: Andreas Dilger Reviewed-by: Sebastien Buisson --- lustre/doc/lfs-find.1 | 35 +++++++++++- lustre/include/lustre/lustreapi.h | 25 ++++++++- lustre/tests/sanity-pfl.sh | 100 +++++++++++++++++++++++++++++++++ lustre/utils/lfs.c | 61 +++++++++++++++++++- lustre/utils/liblustreapi.c | 113 +++++++++++++++++++++++++++++++++++++- 5 files changed, 327 insertions(+), 7 deletions(-) diff --git a/lustre/doc/lfs-find.1 b/lustre/doc/lfs-find.1 index 0fe588b..9ad022e 100644 --- a/lustre/doc/lfs-find.1 +++ b/lustre/doc/lfs-find.1 @@ -14,7 +14,8 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--extension-size|\fB-z\fR [\fB+-\fR]\fIn\fR[\fBKMG\fR]] [[\fB!\fR] \fB--foreign\fR [<\fItype\fR>]] [[\fB!\fR] \fB--gid\fR|\fB-g\fR|\fB--group\fR|\fB-G\fR <\fIgname\fR>|<\fIgid\fR>] - [[\fB!\fR] \fB--layout\fR|\fB-L mdt\fR,\fBraid0\fR,\fBreleased\fR] + [[\fB!\fR] \fB--layout\fR|\fB-L +mdt\fR,\fBraid0\fR,\fBcompress\fR,\fBreleased\fR] [\fB--lazy\fR] [\fB--maxdepth\fR|\fB-D\fI n\fR] [[\fB!\fR] \fB--mdt\fR|\fB--mdt-index\fR|\fB-m\fR <\fIuuid\fR|\fIindex\fR,...>] @@ -38,6 +39,9 @@ lfs-find \- Lustre client utility to list files with specific attributes [[\fB!\fR] \fB--type\fR|\fB-t\fR {\fBbcdflps\fR}] [[\fB!\fR] \fB--uid\fR|\fB-u\fR|\fB--user\fR|\fB-U <\fIuname\fR>|<\fIuid>\fR] + [[\fB!\fR] \fB--compress-type\fR|\fB--compr-type\fR=<\fIcompress-type\fR>] + [[\fB!\fR] \fB--compress-level\fR|\fB--compr-level\fR=[\fB+-\fR]<\fIcompress-level\fR>] + [[\fB!\fR] \fB--compress-chunk\fR|\fB--compr-chunk\fR=[\fB+-\fR]<\fIcompress-chunk\fR>] .SH DESCRIPTION .B lfs find is similar to the standard @@ -139,6 +143,9 @@ HSM-archived files that are not resident in the filesystem. .TP .B mdt Files that have the first data component on an MDT. +.TP +.B compress +Files that contains compress component. .RE .TP .BR --lazy @@ -408,6 +415,32 @@ Extension size is \fIn\fR bytes, or \fBK\fRibi-, \fBM\fRebi-, suffix is given. For composite files, this matches the extension size of any extension component. .TP +.BR --compress-type | --compr-type = \fIcompress-type +Compress algorithm is \fIcompress-type\fR. The file must be a composite +file containing compress component, this matches any component which uses +the specified compress algorithm. +.TP +.BR --compress-level | --compr-level = [ +- ] \fIcompress-level +The compress level which the compress algorithm uses is \fIcompress-level\fR. +The file must be a composite file containing compress component, this matches +any component which uses the specified compress level. If +.BI + compress-level +or +.BI - compress-level +is used, print components with respectively a larger or smaller +\fIcompress-level\fR. +.TP +.BR --compress-chunk | --compr-chunk = [ +- ] \fIcompress-chunk +The compress chunk size (in KiB) which the compress algorithm uses is +\fIcompress-chunk\fR. +The file must be a composite file containing compress component, this matches +any component which uses the specified compress chunk. If +.BI + compress-chunk +or +.BI - compress-chunk +is used, print components with respectively a larger or smaller +\fIcompress-chunk\fR. +.TP .BR --type | -t File has type: \fBb\fRlock, \fBc\fRharacter, \fBd\fRirectory, \fBf\fRile, \fBp\fRipe, sym\fBl\fRink, or \fBs\fRocket. diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 98fb603..8f8ddd6 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -261,7 +261,7 @@ struct find_param { fp_mdt_count_sign:2, fp_blocks_sign:2, fp_ext_size_sign:2, - fp_unused1_sign:2, /* Fields available to use*/ + fp_perm_sign:2, fp_unused2_sign:2, /* Once used we must add */ fp_unused3_sign:2, /* a separate flag field */ fp_unused4_sign:2; /* at end of the struct. */ @@ -327,7 +327,7 @@ struct find_param { fp_lazy:1, fp_newerxy:1, fp_exclude_btime:1, - fp_unused_bit3:1, /* All of these unused bit */ + fp_exclude_perm:1, fp_stop_on_error:1, /* stop iteration on error */ fp_unused_bit5:1, /* are used we need to add */ fp_unused_bit6:1, /* a separate flag field at*/ @@ -404,6 +404,27 @@ struct find_param { unsigned int fp_hash_exflags; /* Print all information (lfs find only) */ char *fp_format_printf_str; + + unsigned int fp_compr_type; + unsigned int fp_compr_lvl; + unsigned int fp_compr_chunk; + + unsigned int fp_check_compr_type:1, + fp_exclude_compr_type:1, + fp_check_compr_lvl:1, + fp_exclude_compr_lvl:1, + fp_check_compr_chunk:1, + fp_exclude_compr_chunk:1, + fp_unused_bit1:1, /* Fields available to use*/ + fp_unused_bit2:1, /* once used, we must add */ + fp_unused_bit3:1, /* a separate flag field */ + fp_unused_bit4:1, /* at */ + fp_unused_bits:22; /* the end of this struct */ + + long long fp_compr_lvl_sign:2, + fp_compr_chunk_sign:2, + fp_unused1_sign:2, /* add sperarate filed */ + fp_unused_signs:58; /* once used up. */ }; int llapi_ostlist(char *path, struct find_param *param); diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 66706ef..ef2e9c9 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -2632,6 +2632,106 @@ test_100c() { } run_test 100c "create compress file with improper compress arguments" +test_100d() { + (( $MDS1_VERSION >= $(version_code 2.14.0.88) )) || + skip "Need MDS >= 2.14.0.88 for compression support" + + local tf=$DIR/$tdir/$tfile + local type="gzip lz4 lz4hc lzo" + local lvl=1 + local cs="64" + local p="$TMP/$TESTSUITE-$TESTNAME.parameters" + + save_lustre_params client "llite.*.enable_compression" > $p + stack_trap "rm -rf $DIR/$tdir; restore_lustre_params < $p" EXIT + $LCTL set_param llite.*.enable_compression=1 + + test_mkdir $DIR/$tdir + $LFS setstripe -c-1 $DIR/$tdir || + error "setstripe upon $DIR/$tdir failed" + + # create compress component files + for tp in $type; do + $LFS setstripe -Eeof -Z $tp:$lvl --compress-chunk=${cs} \ + ${tf}_${tp} || + error "set a compress component in $tf_${tp} failed" + $LFS setstripe -Eeof --comp-flags=nocompr ${tf}_nocompr_${tp} || + error "set a nocompr component in $tf_nocompr_${tp} failed" + $LFS setstripe -Eeof ${tf}_not_${tp} || + error "set a plain file ${tf}_not_${tp} failed" + + lvl=$((lvl + 1)) + cs=$((cs * 2)) + done + + local flg_opts="--comp-flags=compress" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 4 )) || + error "found $found compress (--comp-flags) file != 4" + + flg_opts="--comp-flags=nocompr" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 4 )) || error "found $found nocompr file != 4" + + flg_opts="--comp-flags=^compress" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 9 )) || error "found $found ^compress file != 9" + + flg_opts="-L compress" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 4 )) || error "found $found compress (--layout) file != 4" + + for tp in $type; do + flg_opts="--compr-type=$tp" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 1 )) || error "found $found $tp compress file != 1" + done + + for lvl in $(seq 1 4); do + flg_opts="--compr-level=$lvl" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 1 )) || + error "found $found compress level $lvl file != 1" + + + flg_opts="--compr-level=+$lvl" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + expect=$((4 - lvl)) + (( $found == $expect )) || + error "found $found compress level +$lvl file != $expect" + + flg_opts="--compr-level=-$lvl" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + expect=$lvl + (( $found == $expect )) || + error "found $found compress level -$lvl file != $expect" + done + + cs="64 128 256 512" + local i=1 + for chunk in $cs; do + flg_opts="--compr-chunk=$chunk" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + (( $found == 1 )) || + error "found $found compress $chunk file != 1" + + flg_opts="--compr-chunk=+$chunk" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + expect=$((4 - i)) + (( $found == $expect )) || + error "found $found compress chunk +$chunk file != $expect" + + flg_opts="--compr-chunk=-$chunk" + found=$($LFS find $flg_opts $DIR/$tdir | wc -l) + expect=$i + (( $found == $expect )) || + error "found $found compress chunk -$chunk file != $expect" + + ((i++)) + done +} +run_test 100d "lfs find compress component" + complete $SECONDS check_and_cleanup_lustre exit_status diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 138f53e..efd9421 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -540,10 +540,12 @@ command_t cmdlist[] = { " [[!] --component-start [+-]N[kMGTPE]]\n" " [[!] --component-end|-E [+-]N[kMGTPE]]\n" " [[!] --component-flags {init,stale,prefer,offline,nosync,extension}]\n" + " [[!] --compress-type=\n" + " [[!] --compress-level=[+-]\n" " [[!] --extension-size|--ext-size|-z [+-]N[kMGT]]\n" " [[!] --foreign[=]]\n" " [[!] --gid|-g|--group|-G |]\n" - " [[!] --layout|-L released,raid0,mdt]\n" + " [[!] --layout|-L released,raid0,mdt,compress]\n" " [--maxdepth|-D N] [[!] --mdt-count|-T [+-]]\n" " [[!] --mdt-hash|-H <[^][blm],[^]fnv_1a_64,all_char,crush,...>\n" " [[!] --mdt-index|-m ]\n" @@ -4977,6 +4979,8 @@ static int name2layout(__u32 *layout, char *name) *layout |= LOV_PATTERN_OVERSTRIPING; else if (strcmp(layout_name, "foreign") == 0) *layout |= LOV_PATTERN_FOREIGN; + else if (strcmp(layout_name, "compress") == 0) + *layout |= LOV_PATTERN_COMPRESS; else return -1; } @@ -5013,6 +5017,24 @@ static int lfs_find(int argc, char **argv) { .val = LFS_COMP_START_OPT, .name = "component-start", .has_arg = required_argument }, + { .val = LFS_COMPRESS_TYPE_OPT, + .name = "compr-type", + .has_arg = required_argument }, + { .val = LFS_COMPRESS_TYPE_OPT, + .name = "compress-type", + .has_arg = required_argument }, + { .val = LFS_COMPRESS_LEVEL_OPT, + .name = "compr-level", + .has_arg = required_argument }, + { .val = LFS_COMPRESS_LEVEL_OPT, + .name = "compress-level", + .has_arg = required_argument }, + { .val = LFS_COMPRESS_CHUNK_OPT, + .name = "compr-chunk", + .has_arg = required_argument }, + { .val = LFS_COMPRESS_CHUNK_OPT, + .name = "compress-chunk", + .has_arg = required_argument }, { .val = LFS_MIRROR_STATE_OPT, .name = "mirror-state", .has_arg = required_argument }, { .val = LFS_NEWERXY_OPT, @@ -5286,6 +5308,43 @@ static int lfs_find(int argc, char **argv) param.fp_check_comp_start = 1; param.fp_exclude_comp_start = !!neg_opt; break; + case LFS_COMPRESS_TYPE_OPT: + rc = llapi_parse_compress_type(optarg, + ¶m.fp_compr_type, + NULL); + if (rc) { + fprintf(stderr, + "error: unknown compress type '%s'\n", + optarg); + goto err; + } + param.fp_check_compr_type = 1; + param.fp_exclude_compr_type = !!neg_opt; + break; + case LFS_COMPRESS_LEVEL_OPT: + if (optarg[0] == '+') { + param.fp_compr_lvl_sign = -1; + optarg++; + } else if (optarg[0] == '-') { + param.fp_compr_lvl_sign = 1; + optarg++; + } + param.fp_compr_lvl = atoi(optarg); + param.fp_check_compr_lvl = 1; + param.fp_exclude_compr_lvl = !!neg_opt; + break; + case LFS_COMPRESS_CHUNK_OPT: + if (optarg[0] == '+') { + param.fp_compr_chunk_sign = -1; + optarg++; + } else if (optarg[0] == '-') { + param.fp_compr_chunk_sign = 1; + optarg++; + } + param.fp_compr_chunk = atoi(optarg); + param.fp_check_compr_chunk = 1; + param.fp_exclude_compr_chunk = !!neg_opt; + break; case LFS_MIRROR_STATE_OPT: rc = mirror_str2state(optarg, ¶m.fp_mirror_state, ¶m.fp_mirror_neg_state); diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 0e7683e..9118a28 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -3367,13 +3367,14 @@ static int find_value_cmp(unsigned long long file, unsigned long long limit, bool mds) { int ret = -1; - if (sign > 0) { /* Drop the fraction of margin (of days or size). */ if (file + margin <= limit) ret = mds ? 0 : 1; } else if (sign == 0) { - if (file <= limit && file + margin > limit) + if (file == limit) + ret = mds ? 0 : 1; + else if (file < limit && file + margin > limit) ret = mds ? 0 : 1; else if (file + margin <= limit) ret = mds ? 0 : -1; @@ -4355,6 +4356,104 @@ static __u32 find_get_stripe_count(struct find_param *param) return stripe_count; } +static int find_check_compress(struct find_param *param) +{ + struct lov_comp_md_v1 *comp_v1 = NULL; + struct lov_user_md_v1 *v1 = ¶m->fp_lmd->lmd_lmm; + struct lov_comp_md_entry_v1 *entry; + int i, j; + bool valid = false; + bool found_type = false, found_lvl = false, found_chunk = false; + unsigned long long val; + + if (v1->lmm_magic != LOV_USER_MAGIC_COMP_V1) + return -1; + + comp_v1 = (struct lov_comp_md_v1 *)v1; + + for (i = 0; i < comp_v1->lcm_entry_count; i++) { + entry = &comp_v1->lcm_entries[i]; + + /** + * if it's not a compress file, don't count it as a valid + * candidate. + */ + if (!(lov_comp_entry(comp_v1, i)->lmm_pattern & + LOV_PATTERN_COMPRESS)) + continue; + + valid = true; + + if (param->fp_check_compr_type && + (entry->lcme_compr_type == param->fp_compr_type)) + found_type = true; + + if (param->fp_check_compr_lvl) { + for (j = 0; j < ARRAY_SIZE(compr_type_table); j++) { + if (compr_type_table[j].ctn_compr_type != + entry->lcme_compr_type) + continue; + + val = entry->lcme_compr_lvl; + if (compr_type_table[j].ctn_to_compr_level) + val = compr_type_table[j]. + ctn_to_compr_level( + entry->lcme_compr_lvl); + + if (find_value_cmp(val, + param->fp_compr_lvl, + param->fp_compr_lvl_sign, + param->fp_exclude_compr_lvl, + 0, 0) == 1) { + found_lvl = true; + break; + } + } + } + + if (param->fp_check_compr_chunk) { + for (j = 0; j < ARRAY_SIZE(compr_type_table); j++) { + if (compr_type_table[j].ctn_compr_type != + entry->lcme_compr_type) + continue; + + /* chunk_size_in_KiB = 2^chunk_log_bits * 64 */ + val = 1 << (entry->lcme_compr_chunk_log_bits + + 6); + if (find_value_cmp(val, + param->fp_compr_chunk, + param->fp_compr_chunk_sign, + param->fp_exclude_compr_chunk, + 0, 0) == 1) { + found_chunk = true; + break; + } + } + } + + if ((param->fp_check_compr_type && found_type) || + (param->fp_check_compr_lvl && found_lvl) || + (param->fp_check_compr_chunk && found_chunk)) + break; + } + + if (!valid) + return -1; + + if (param->fp_check_compr_type && + ((found_type && !param->fp_exclude_compr_type) || + (!found_type && param->fp_exclude_compr_type))) + return 1; + + if (param->fp_check_compr_lvl && found_lvl) + return 1; + + if (param->fp_check_compr_chunk && found_chunk) + return 1; + + return -1; +} + #define LOV_PATTERN_INVALID 0xFFFFFFFF static int find_check_layout(struct find_param *param) @@ -4608,7 +4707,8 @@ static bool find_check_lmm_info(struct find_param *param) param->fp_check_comp_start || param->fp_check_comp_flags || param->fp_check_mirror_count || param->fp_check_foreign || param->fp_check_mirror_state || param->fp_check_ext_size || - param->fp_check_projid; + param->fp_check_projid || param->fp_check_compr_type || + param->fp_check_compr_lvl || param->fp_check_compr_chunk; } /* @@ -5455,6 +5555,13 @@ static int cb_find_init(char *path, DIR *parent, DIR **dirp, goto decided; } + if (param->fp_check_compr_type || param->fp_check_compr_lvl || + param->fp_check_compr_chunk) { + decision = find_check_compress(param); + if (decision == -1) + goto decided; + } + /* If an OBD UUID is specified but none matches, skip this file. */ if ((param->fp_obd_uuid && param->fp_obd_index == OBD_NOT_FOUND) || (param->fp_mdt_uuid && param->fp_mdt_index == OBD_NOT_FOUND)) -- 1.8.3.1