From bd462ce8e4301f3d743adddf12fcb93b49c69da7 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Mon, 29 Jan 2024 18:26:24 -0800 Subject: [PATCH] EX-7795 tests: add sanity-compr test for dir compression This patch adds a sanity-compr test to validate that we get directory space usage reduction with compression. Change-Id: I16f3a3f1e413e4884b3973829df36500667271ce Test-Parameters: trivial testlist=sanity-compr env=ONLY="1007 1008",ONLY_REPEAT=3 Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53855 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Colin Faber Reviewed-by: Andreas Dilger --- lustre/tests/sanity-compr.sh | 138 +++++++++++++++++++++++++++++++++++++++-- lustre/tests/test-framework.sh | 10 ++- 2 files changed, 141 insertions(+), 7 deletions(-) diff --git a/lustre/tests/sanity-compr.sh b/lustre/tests/sanity-compr.sh index 6e38b70..382fef6 100644 --- a/lustre/tests/sanity-compr.sh +++ b/lustre/tests/sanity-compr.sh @@ -969,6 +969,9 @@ test_1007() { local chunksize # lzo does not grok a compression level, add here other such algs local type_nolvl="lzo" + local total_estimates=0 + local failed_estimates=0 + local failed_margin=50 # Fail test if source size changes so we catch this # Source should be a few MiB in size @@ -1009,23 +1012,148 @@ test_1007() { local csdc_size=$(du -sk $tf | awk '{print $1}') local margin=5 + local real_margin=0 echo "Estimated compressed size: $estimated_size KiB" echo "CSDC size: $csdc_size KiB" - (( csdc_size > ${estimated_size%%.*} * (100 - margin) / 100 && - csdc_size < ${estimated_size%%.*} * (100 + margin) / 100 )) || + real_margin=$(( csdc_size * 100 / ${estimated_size%%.*} )) + (( real_margin > 100 )) && (( real_margin -= 100 )) || + real_margin=$(( 100 - real_margin )) + (( real_margin <= margin )) || { local error_msg="estimated compressed size: " error_msg+="$estimated_size KiB, " error_msg+="got: $csdc_size KiB, " - error_msg+="margin exceeded ${margin}%" - error_ignore EX-7795 "$error_msg" + error_msg+="margin ${real_margin}% exceeded ${margin}%" + error_ignore_no_dump EX-7795 "$error_msg" + + (( failed_estimates += 1 )) } + + (( total_estimates += 1 )) done # chunksize [[ -n $has_level ]] || break done # compr_level done # compr_type + + (( failed_estimates <= total_estimates * failed_margin / 100 )) || + error "failed estimates > ${failed_margin}% of total estimates" +} +run_test 1007 "validate file space usage reduction with compression" + +test_1008() { + (( $OST1_VERSION >= $(version_code 2.14.0-ddn128) )) || + skip "Need OST >= 2.14.0-ddn127-9-g6c4c4d7599 for good compr" + + # The ll_compression_scan tool requires the lz4, lzop, and gzip + # utilities to be installed in order to test those compression types. + # (lzop is the command line utility for lzo compression) + local -a compr_types + local utility + local rc + + for utility in ll_compression_scan lz4 lzop gzip; do + which $utility &>/dev/null + rc=$? + + case $utility-$rc in + lz4-0) compr_types+=(lz4 lz4fast) ;; + lzop-0) compr_types+=(lzo) ;; + gzip-0) compr_types+=(gzip) ;; + ll_compression_scan-1) error "'$utility' is not installed";; + *-1) echo "'$utility' not found"; continue;; + esac + done + + [[ -n "${compr_types}" ]] || error "no compression tools installed" + + local compr_levels=( + 1 + 6 + 9 + ) + + local chunksizes=( + 64 + 128 + 1024 + ) + + local compr_type + local compr_level + local chunksize + # lzo does not grok a compression level, add here other such algs + local type_nolvl="lzo" + local total_estimates=0 + local failed_estimates=0 + local failed_margin=50 + + local td=$DIR/$tdir + local source="/etc/ /bin/" + + stack_trap "rm -rf $td; disable_compression" + enable_compression + + for compr_type in ${compr_types[@]}; do + for compr_level in ${compr_levels[@]}; do + for chunksize in "${chunksizes[@]}"; do + rm -rf $td + mkdir $td || error "mkdir $td failed" + + local has_level="" + [[ "$type_nolvl" =~ "$compr_type" ]] || has_level="yes" + + local compr_info="algorithm $compr_type" + [[ -z $has_level ]] || compr_info+=", level $compr_level" + compr_info+=", and chunsksize ${chunksize}K" + + echo "Compression, using $compr_info" + $LFS setstripe -E -1 \ + -Z ${compr_type}${has_level:+":${compr_level}"} \ + --compress-chunk=$chunksize $td || + error "setstripe on $td failed with $compr_info" + + cp -a $source $td || error "copy $source to $td failed" + # Sync to disk and drop cache + sync; echo 3 > /proc/sys/vm/drop_caches + + local scan_cmd="ll_compression_scan -m 0 -w -q -z $compr_type" + [[ -z $has_level ]] || scan_cmd+=" -l $compr_level" + scan_cmd+=" -c $chunksize" + + local estimated_size=$($scan_cmd $source | + awk '/Estimated compressed size/ { print $7 }') + [[ -n "$estimated_size" ]] || error "no compression estimate" + estimated_size=$(bc -l <<< "$estimated_size * 1024") + local csdc_size=$(du -sk $td | awk '{print $1}') + + local margin=5 + local real_margin=0 + echo "Estimated compressed size: $estimated_size KiB" + echo "CSDC size: $csdc_size KiB" + real_margin=$(( csdc_size * 100 / ${estimated_size%%.*} )) + (( real_margin > 100 )) && (( real_margin -= 100 )) || + real_margin=$(( 100 - real_margin )) + (( real_margin <= margin )) || + { + local error_msg="estimated compressed size: " + error_msg+="$estimated_size KiB, " + error_msg+="got: $csdc_size KiB, " + error_msg+="margin ${real_margin}% exceeded ${margin}%" + error_ignore_no_dump EX-7795 "$error_msg" + + (( failed_estimates += 1 )) + } + + (( total_estimates += 1 )) + done # chunksize + [[ -n $has_level ]] || break + done # compr_level + done # compr_type + + (( failed_estimates <= total_estimates * failed_margin / 100 )) || + error "failed estimates > ${failed_margin}% of total estimates" } -run_test 1007 "validate space usage reduction with compression" +run_test 1008 "validate directory space usage reduction with compression" complete_test $SECONDS check_and_cleanup_lustre diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 6bb202c..478daf7 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -6549,13 +6549,13 @@ report_error() { fi log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " - (print_stack_trace 2) >&2 - mkdir -p $LOGDIR # We need to dump the logs on all nodes if $dump; then + (print_stack_trace 2) >&2 gather_logs $(comma_list $(nodes_list)) fi + mkdir -p $LOGDIR debugrestore [ "$TESTSUITELOG" ] && echo "$TESTSUITE: $TYPE: $TESTNAME $@" >> $TESTSUITELOG @@ -6644,6 +6644,12 @@ error_ignore() { report_error "$@" } +error_ignore_no_dump() { + local TYPE="IGNORE ($1)" + shift + report_error false "$@" +} + error_and_remount() { report_error "$@" remount_client $MOUNT -- 1.8.3.1