From 15387f845b6a3f7a68dc3913c39c9d44dfd719a1 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Mon, 8 Jan 2024 23:09:10 -0800 Subject: [PATCH] EX-7795 tests: add sanity-compr test for compression This patch adds a sanity-compr test to validate that we get space usage reduction with compression. The test uses ll_compression_scan tool to calculate the compressed size of the source file and compares it with the size of the Lustre CSDC compressed file. Test-Parameters: trivial env=ONLY=1007 testlist=sanity-compr Change-Id: Icf763331205a3e937b794f90444f756fc59f9050 Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52895 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Sarah Liu Reviewed-by: Andreas Dilger --- lustre.spec.in | 3 +- lustre/scripts/Makefile.am | 3 +- lustre/tests/sanity-compr.sh | 113 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/lustre.spec.in b/lustre.spec.in index d1dc7fc..f500407 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -933,9 +933,10 @@ fi %{_bindir}/lustre_req_history %{_bindir}/remove_changelog %{_bindir}/remove_updatelog -%{_bindir}/ll_compression_scan %endif +%{_bindir}/ll_compression_scan + %{_bindir}/llobdstat %{_bindir}/lljobstat %{_bindir}/llstat diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am index cc516c9..1754e1f 100644 --- a/lustre/scripts/Makefile.am +++ b/lustre/scripts/Makefile.am @@ -57,12 +57,11 @@ endif if UTILS sbin_SCRIPTS += ldev lustre_routes_config lustre_routes_conversion -bin_SCRIPTS = lfs_migrate +bin_SCRIPTS = lfs_migrate ll_compression_scan if SERVER sbin_SCRIPTS += $(genscripts) lc_mon lhbadm lc_servip bin_SCRIPTS += lustre_req_history remove_changelog remove_updatelog -bin_SCRIPTS += ll_compression_scan hadir = $(sysconfdir)/ha.d/resource.d ha_SCRIPTS = Lustre.ha_v2 diff --git a/lustre/tests/sanity-compr.sh b/lustre/tests/sanity-compr.sh index 2e9ae08..ce06081 100644 --- a/lustre/tests/sanity-compr.sh +++ b/lustre/tests/sanity-compr.sh @@ -896,6 +896,119 @@ test_1006() { } run_test 1006 "racing writes with compressed layout" +test_1007() { + (( MDS1_VERSION >= $(version_code 2.14.0-ddn128) )) || + skip "Need MDS version at least 2.14.0-ddn128" + + # The ll_compression_scan tool requires the lz4, lzop, and gzip + # utilities to be installed in order to test those compression types. + # (lzop is the command line utility for lzo compression) + local -a compr_types + local utility + local rc + + for utility in ll_compression_scan lz4 lzop gzip; do + which $utility &>/dev/null + rc=$? + + case $utility-$rc in + lz4-0) compr_types+=(lz4 lz4fast) ;; + lzop-0) compr_types+=(lzo) ;; + gzip-0) compr_types+=(gzip) ;; + ll_compression_scan-1) error "'$utility' is not installed";; + *-1) echo "'$utility' not found"; continue;; + esac + done + + [[ -n "${compr_types}" ]] || error "no compression tools installed" + + local tf=$DIR/$tfile + local hdf=$LUSTRE/tests/AMSR_E_L3_DailyOcean_V05_20111003.hdf + local tmp_hdf=$TMP/$tfile.hdf + local source=$tmp_hdf + + if [[ -f $hdf.bz2 ]] && type -p bzcat >/dev/null; then + bzcat $hdf.bz2 > $tmp_hdf + elif [[ -f $hdf.bz2 ]] && type -p bunzip2 >/dev/null; then + cp $hdf.bz2 $tmp_hdf.bz2 || error "cp $tmp_hdf.bz2" + bunzip2 $tmp_hdf.bz2 || error "bunzip2 $tmp_hdf.bz2" + else + skip_env "bunzip2 is not installed" + fi + + local compr_levels=( + 1 + 6 + 9 + ) + + local chunksizes=( + 64 + 128 + 1024 + ) + + local compr_type + local compr_level + local chunksize + # lzo does not grok a compression level, add here other such algs + local type_nolvl="lzo" + + # Fail test if source size changes so we catch this + # Source should be a few MiB in size + $CHECKSTAT -s 14625450 $source || error "checkstat wrong size" + + stack_trap "rm -f $tf; disable_compression" + enable_compression + + for compr_type in ${compr_types[@]}; do + for compr_level in ${compr_levels[@]}; do + for chunksize in "${chunksizes[@]}"; do + rm -f $tf + local has_level="" + [[ "$type_nolvl" =~ "$compr_type" ]] || has_level="yes" + + local compr_info="algorithm $compr_type" + [[ -z $has_level ]] || compr_info+=", level $compr_level" + compr_info+=", and chunsksize ${chunksize}K" + + echo "Compression, using $compr_info" + $LFS setstripe -E -1 \ + -Z ${compr_type}${has_level:+":${compr_level}"} \ + --compress-chunk=$chunksize $tf || + error "setstripe on $tf failed with $compr_info" + + cp -a $source $tf || error "copy $source to $tf failed" + # Sync to disk and drop cache + sync; echo 3 > /proc/sys/vm/drop_caches + + local scan_cmd="ll_compression_scan -w -q -z $compr_type" + [[ -z $has_level ]] || scan_cmd+=" -l $compr_level" + scan_cmd+=" -c $chunksize" + + local estimated_size=$($scan_cmd $source | + awk '/Estimated compressed size/{print $7}') + local csdc_size=$(du -sk $tf | awk '{print $1}') + + local margin=5 + echo "Estimated compressed size: $estimated_size KiB" + echo "CSDC size: $csdc_size KiB" + (( csdc_size > ${estimated_size%%.*} * (100 - margin) / 100 && + csdc_size < ${estimated_size%%.*} * (100 + margin) / 100 )) || + { + local error_msg="estimated compressed size: " + error_msg+="$estimated_size KiB, " + error_msg+="got: $csdc_size KiB, " + error_msg+="margin exceeded ${margin}%" + error_ignore EX-7795 "$error_msg" + } + done # chunksize + [[ -n $has_level ]] || break + done # compr_level + done # compr_type +} +run_test 1007 "validate space usage reduction with compression" + complete_test $SECONDS check_and_cleanup_lustre declare -a logs=($ONLY) -- 1.8.3.1