EX-7795 tests: add sanity-compr test for compression

This patch adds a sanity-compr test to validate that
we get space usage reduction with compression.
The test uses ll_compression_scan tool to calculate
the compressed size of the source file and compares
it with the size of the Lustre CSDC compressed file.

Test-Parameters: trivial env=ONLY=1007 testlist=sanity-compr

Change-Id: Icf763331205a3e937b794f90444f756fc59f9050
Signed-off-by: Jian Yu <yujian@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52895
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Sarah Liu <sarah@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre.spec.in b/lustre.spec.in
index d1dc7fc..f500407 100644 (file)
--- a/lustre.spec.in
+++ b/lustre.spec.in
@@ -933,9 +933,10 @@ fi
 %{_bindir}/lustre_req_history
 %{_bindir}/remove_changelog
 %{_bindir}/remove_updatelog
-%{_bindir}/ll_compression_scan
 %endif
 
+%{_bindir}/ll_compression_scan
+
 %{_bindir}/llobdstat
 %{_bindir}/lljobstat
 %{_bindir}/llstat

diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am
index cc516c9..1754e1f 100644 (file)
--- a/lustre/scripts/Makefile.am
+++ b/lustre/scripts/Makefile.am
@@ -57,12 +57,11 @@ endif
 
 if UTILS
 sbin_SCRIPTS += ldev lustre_routes_config lustre_routes_conversion
-bin_SCRIPTS   = lfs_migrate
+bin_SCRIPTS   = lfs_migrate ll_compression_scan
 
 if SERVER
 sbin_SCRIPTS += $(genscripts) lc_mon lhbadm lc_servip
 bin_SCRIPTS  += lustre_req_history remove_changelog remove_updatelog
-bin_SCRIPTS  += ll_compression_scan
 
 hadir = $(sysconfdir)/ha.d/resource.d
 ha_SCRIPTS = Lustre.ha_v2

diff --git a/lustre/tests/sanity-compr.sh b/lustre/tests/sanity-compr.sh
index 2e9ae08..ce06081 100644 (file)
--- a/lustre/tests/sanity-compr.sh
+++ b/lustre/tests/sanity-compr.sh
@@ -896,6 +896,119 @@ test_1006() {
 }
 run_test 1006 "racing writes with compressed layout"
 
+test_1007() {
+       (( MDS1_VERSION >= $(version_code 2.14.0-ddn128) )) ||
+               skip "Need MDS version at least 2.14.0-ddn128"
+
+       # The ll_compression_scan tool requires the lz4, lzop, and gzip
+       # utilities to be installed in order to test those compression types.
+       # (lzop is the command line utility for lzo compression)
+       local -a compr_types
+       local utility
+       local rc
+
+       for utility in ll_compression_scan lz4 lzop gzip; do
+               which $utility &>/dev/null
+               rc=$?
+
+               case $utility-$rc in
+               lz4-0) compr_types+=(lz4 lz4fast) ;;
+               lzop-0) compr_types+=(lzo) ;;
+               gzip-0) compr_types+=(gzip) ;;
+               ll_compression_scan-1) error "'$utility' is not installed";;
+               *-1) echo "'$utility' not found"; continue;;
+               esac
+       done
+
+       [[ -n "${compr_types}" ]] || error "no compression tools installed"
+
+       local tf=$DIR/$tfile
+       local hdf=$LUSTRE/tests/AMSR_E_L3_DailyOcean_V05_20111003.hdf
+       local tmp_hdf=$TMP/$tfile.hdf
+       local source=$tmp_hdf
+
+       if [[ -f $hdf.bz2 ]] && type -p bzcat >/dev/null; then
+               bzcat $hdf.bz2 > $tmp_hdf
+       elif [[ -f $hdf.bz2 ]] && type -p bunzip2 >/dev/null; then
+               cp $hdf.bz2 $tmp_hdf.bz2 || error "cp $tmp_hdf.bz2"
+               bunzip2 $tmp_hdf.bz2 || error "bunzip2 $tmp_hdf.bz2"
+       else
+               skip_env "bunzip2 is not installed"
+       fi
+
+       local compr_levels=(
+               1
+               6
+               9
+       )
+
+       local chunksizes=(
+               64
+               128
+               1024
+       )
+
+       local compr_type
+       local compr_level
+       local chunksize
+       # lzo does not grok a compression level, add here other such algs
+       local type_nolvl="lzo"
+
+       # Fail test if source size changes so we catch this
+       # Source should be a few MiB in size
+       $CHECKSTAT -s 14625450 $source || error "checkstat wrong size"
+
+       stack_trap "rm -f $tf; disable_compression"
+       enable_compression
+
+       for compr_type in ${compr_types[@]}; do
+       for compr_level in ${compr_levels[@]}; do
+       for chunksize in "${chunksizes[@]}"; do
+               rm -f $tf
+               local has_level=""
+               [[ "$type_nolvl" =~ "$compr_type" ]] || has_level="yes"
+
+               local compr_info="algorithm $compr_type"
+               [[ -z $has_level ]] || compr_info+=", level $compr_level"
+               compr_info+=", and chunsksize ${chunksize}K"
+
+               echo "Compression, using $compr_info"
+               $LFS setstripe -E -1 \
+                       -Z ${compr_type}${has_level:+":${compr_level}"} \
+                       --compress-chunk=$chunksize $tf ||
+                       error "setstripe on $tf failed with $compr_info"
+
+               cp -a $source $tf || error "copy $source to $tf failed"
+               # Sync to disk and drop cache
+               sync; echo 3 > /proc/sys/vm/drop_caches
+
+               local scan_cmd="ll_compression_scan -w -q -z $compr_type"
+               [[ -z $has_level ]] || scan_cmd+=" -l $compr_level"
+               scan_cmd+=" -c $chunksize"
+
+               local estimated_size=$($scan_cmd $source |
+                       awk '/Estimated compressed size/{print $7}')
+               local csdc_size=$(du -sk $tf | awk '{print $1}')
+
+               local margin=5
+               echo "Estimated compressed size: $estimated_size KiB"
+               echo "CSDC size: $csdc_size KiB"
+               (( csdc_size > ${estimated_size%%.*} * (100 - margin) / 100 &&
+                  csdc_size < ${estimated_size%%.*} * (100 + margin) / 100 )) ||
+               {
+                       local error_msg="estimated compressed size: "
+                       error_msg+="$estimated_size KiB, "
+                       error_msg+="got: $csdc_size KiB, "
+                       error_msg+="margin exceeded ${margin}%"
+                       error_ignore EX-7795 "$error_msg"
+               }
+       done # chunksize
+       [[ -n $has_level ]] || break
+       done # compr_level
+       done # compr_type
+}
+run_test 1007 "validate space usage reduction with compression"
+
 complete_test $SECONDS
 check_and_cleanup_lustre
 declare -a logs=($ONLY)