From 9fa097adfc319ffa5ab20034d55a1f0020c3bd9d Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Tue, 4 May 2021 10:41:23 -0700 Subject: [PATCH] EX-2659 tests: add sanity-lipe.sh to test LiPE utilities This patch adds sanity-lipe.sh test script to test lipe_find and lipe_scan utilities for LiPE. Lustre-change: https://review.whamcloud.com/42151 Lustre-commit: 5d67c987c8d2dc393b1e0952fe01d33978efdea0 Test-Parameters: trivial testlist=sanity-lipe Change-Id: I69d82f7e3675becb4e38915ff363e853d0accb77 Signed-off-by: Jian Yu Reviewed-by: James Nunez Reviewed-by: John L. Hammond Reviewed-on: https://review.whamcloud.com/43536 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Li Xi --- lipe/lipe.spec.in | 2 +- lustre/tests/Makefile.am | 2 +- lustre/tests/sanity-lipe.sh | 1342 +++++++++++++++++++++++++++++++++++ lustre/tests/test-groups/regression | 1 + 4 files changed, 1345 insertions(+), 2 deletions(-) create mode 100644 lustre/tests/sanity-lipe.sh diff --git a/lipe/lipe.spec.in b/lipe/lipe.spec.in index c77b125..9e1114d 100644 --- a/lipe/lipe.spec.in +++ b/lipe/lipe.spec.in @@ -30,7 +30,7 @@ Group: Applications/System Source0: @PACKAGE@-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires: lipe-pylustre = %{version}-%{release} -Requires: rsync json-c libyaml +Requires: rsync json-c libyaml PyYAML python2-filelock python-dateutil Requires: e2fsprogs >= 1.42.13.wc6 Provides: lipe = %{version}-%{release} %if %{with systemd} diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index bd668fe..91381ce 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -35,7 +35,7 @@ noinst_SCRIPTS += mds-survey.sh parallel-scale-nfs.sh large-lun.sh noinst_SCRIPTS += parallel-scale-nfsv3.sh parallel-scale-nfsv4.sh noinst_SCRIPTS += setup-cifs.sh parallel-scale-cifs.sh noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh ha.sh pjdfstest.sh -noinst_SCRIPTS += sanity-lfsck.sh lfsck-performance.sh +noinst_SCRIPTS += sanity-lfsck.sh lfsck-performance.sh sanity-lipe.sh noinst_SCRIPTS += resolveip noinst_SCRIPTS += sanity-hsm.sh sanity-lsnapshot.sh sanity-pfl.sh sanity-flr.sh noinst_SCRIPTS += sanity-dom.sh sanity-pcc.sh dom-performance.sh sanity-lnet.sh diff --git a/lustre/tests/sanity-lipe.sh b/lustre/tests/sanity-lipe.sh new file mode 100644 index 0000000..3197c1b --- /dev/null +++ b/lustre/tests/sanity-lipe.sh @@ -0,0 +1,1342 @@ +#!/bin/bash +# +# Tests for lipe_find and lipe_scan. +# +# lipe_find - search for files on Lustre devices or directories +# lipe_scan - fast scan tool based on LiPE (Lustre integrated Policy Engine) +set -e + +ONLY=${ONLY:-"$*"} + +LUSTRE=${LUSTRE:-$(dirname $0)/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging + +# bug number for skipped test: +ALWAYS_EXCEPT="$SANITY_LIPE_EXCEPT " +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +# FIXME to support DNE +(( MDSCOUNT == 1 )) || skip_env "need 1 MDT" +(( OSTCOUNT >= 2 )) || skip_env "need at least 2 OSTs" + +[[ $(facet_fstype mds1) = ldiskfs ]] || skip_env "need ldiskfs on MDS" + +! remote_mds_nodsh || skip_env "remote MDS with nodsh" +! remote_ost_nodsh || skip_env "remote OSS with nodsh" + +# check if lipe_find and lipe_scan are installed on MDS(s) +for t in lipe_find lipe_scan; do + do_nodes $(comma_list $(all_mdts_nodes)) "which $t" &>/dev/null || + skip_env "$t is not installed on MDS" +done + +# lipe_find configuration +LIPE_FIND="lipe_find" +LIPE_FIND_LOG_DIR=${LIPE_FIND_LOG_DIR:-"/var/log/lipe_find"} +LIPE_FIND_VERBOSE=${LIPE_FIND_VERBOSE:-false} +LIPE_FIND_USE_MOUNT=${LIPE_FIND_USE_MOUNT:-false} + +LIPE_POOL1=${LIPE_POOL1:-"fast"} +LIPE_POOL2=${LIPE_POOL2:-"slow"} + +build_test_filter +check_and_setup_lustre + +# create OST pools +create_ost_pools() { + pool_add $LIPE_POOL1 || error "failed to create OST pool '$LIPE_POOL1'" + pool_add_targets $LIPE_POOL1 0 $((OSTCOUNT / 2 - 1)) || + error "failed to add targets to OST pool '$LIPE_POOL1'" + pool_add $LIPE_POOL2 || error "failed to create OST pool '$LIPE_POOL2'" + pool_add_targets $LIPE_POOL2 $((OSTCOUNT / 2)) $((OSTCOUNT - 1)) || + error "failed to add targets to OST pool '$LIPE_POOL2'" +} + +# Convert file type string to type option or constant used in lipe_find +convert_file_type() { + local arg="$1" + local str="$2" + local option + local type + + ! [[ $arg != option && $arg != constant ]] || return 0 + [[ $arg = option ]] && option=true || option=false + + case "$str" in + regular*) $option && type="f" || type="S_IFREG" ;; + block*) $option && type="b" || type="S_IFBLK" ;; + character*) $option && type="c" || type="S_IFCHR" ;; + directory) $option && type="d" || type="S_IFDIR" ;; + symbolic*) $option && type="l" || type="S_IFLNK" ;; + fifo) $option && type="p" || type="S_IFIFO" ;; + socket) $option && type="s" || type="S_IFSOCK" ;; + *) return ;; + esac + + echo -n "$type" +} + +declare -A type_to_mode=( + [p]=0010666 + [c]=0020666 + [d]=0040666 + [b]=0060666 + [f]=0100666 + [l]=0120666 + [s]=0140666 +) + +create_special_files() { + local file_name="$1" + local file + local device + local loop_dev="/dev/loop0" + local cmd + local t + + [[ -n "$file_name" ]] || error "file_name was not specified" + + for t in ${!type_to_mode[@]}; do + file="$file_name.$t" + + case $t in + c) device=$(stat --format=%D /dev/null) ;; + b) [[ -e $loop_dev ]] || { + mknod -m660 $loop_dev b 7 0 + chown root.disk $loop_dev + } + device=$(stat --format=%D $loop_dev) + ;; + *) device=0 ;; + esac + + cmd="mcreate --mode=${type_to_mode[$t]} --device=$device $file" + echo $cmd + $cmd || error "mcreate '$file' failed" + done +} + +create_regular_files() { + local i + local pool + if [[ "$1" = "--with-pool" ]]; then + shift + pool=yes + fi + + local file="$1" + [[ -n "$file" ]] || error "file was not specified" + + # create OST pools + [[ -z "$pool" ]] || create_ost_pools + + # create a regular file + cmd="$LFS setstripe -c -1 -i -1 -S 4M ${pool:+-p $LIPE_POOL1 }$file.1" + echo $cmd + $cmd || error "$LFS setstripe '$file.1' failed" + + # create a PFL file + cmd="$LFS setstripe -E 1M -c 1 ${pool:+-p $LIPE_POOL1 }" + cmd+="--comp-flags=prefer -E eof -c -1 ${pool:+-p $LIPE_POOL2 }$file.2" + echo $cmd + $cmd || error "$LFS setstripe '$file.2' failed" + + # create a FLR file + cmd="$LFS mirror create -N -E 1M -c -1 ${pool:+-p $LIPE_POOL1 }" + cmd+="-E eof -S 4M ${pool:+-p $LIPE_POOL2 }" + cmd+="-N ${pool:+-p $LIPE_POOL1 }$file.3" + echo $cmd + $cmd || error "$LFS mirror create '$file.3' failed" + + # create a DoM file + cmd="$LFS setstripe -E 1M -L mdt -E eof $file.4" + echo $cmd + $cmd || error "$LFS setstripe '$file.4' failed" + + # create hard links to file + for i in {1..3}; do + cmd="ln $file.1 $file.1.link$i" + echo $cmd + $cmd || error "ln $file.1 $file.1.link$i failed" + done + + # write some data into the files + for i in {1..4}; do + dd bs=${i}M count=1 iflag=fullblock oflag=sync \ + if=/dev/urandom of=$file.$i || + error "failed to write to $file.$i" + done + sync; sync_all_data; sync +} + +create_lipe_find_files() { + local pool + if [[ "$1" = "--with-pool" ]]; then + shift + pool=yes + fi + + local file_name="$1" + [[ -n "$file_name" ]] || error "file_name was not specified" + + create_regular_files ${pool:+--with-pool }$file_name + create_special_files $file_name +} + +do_lipe_find_facet() { + local facet="$1" + local action="$2" + local device="$(facet_device $facet)" + local quiet + + $LIPE_FIND_VERBOSE || quiet=yes + ! $LIPE_FIND_USE_MOUNT || device=$MOUNT + + # lipe_find fails if run twice in the same wall clock second. + VERBOSE=false do_facet $facet "sync; [[ ! -e $LIPE_FIND_LOG_DIR ]] ||" \ + "find $LIPE_FIND_LOG_DIR -mindepth 1 -delete" + do_facet $facet "$LIPE_FIND $action $device ${@:3}" \ + "${quiet:+2>/dev/null}" +} + +expect_fid() { + local fid="$1" + local which="${2:-true}" + + if $which; then + grep --fixed-strings "$fid" || error "expected FID '$fid'" + else + ! grep --fixed-strings "$fid" || error "unexpected FID '$fid'" + fi +} + +expect_file() { + local file="$1" + local which="${2:-true}" + local fid + + fid=$($LFS path2fid "$file") + [[ -n "$fid" ]] || error "cannot get FID of '$file'" + + expect_fid "$fid" "$which" +} + +lipe_find_expect_file() { + local file="$1" + local which="$2" + + do_lipe_find_facet mds1 -print-fid "${@:3}" | expect_file "$file" "$which" +} + +lipe_find_expect_fid() { + local fid="$1" + local which="$2" + + do_lipe_find_facet mds1 -print-fid "${@:3}" | expect_fid "$fid" "$which" +} + +lipe_find_max_stripe_size() { + local file="$1" + local stripe_size + local max_stripe_size=$($LFS getstripe -S $file) + local ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | + tr '\n' ' ')) + local id + + if (( ${#ids[@]} == 0 )); then + echo -n $max_stripe_size + return + fi + + for id in ${ids[@]}; do + stripe_size=$($LFS getstripe -I$id -S $file) + (( stripe_size <= max_stripe_size)) || + max_stripe_size=$stripe_size + done + + echo -n $max_stripe_size +} + +# lipe_find test cases +test_1() { + do_lipe_find_facet mds1 -print-fid || error "cannot run lipe_find" +} +run_test 1 "lipe_find works" + +test_2() { + local td=$DIR/$tdir + local tf=$td/$tfile + + mkdir $td || error "mkdir '$td' failed" + touch $tf || error "touch '$tf' failed" + + lipe_find_expect_file $td + lipe_find_expect_file $tf +} +run_test 2 "lipe_find finds dir and file" + +test_3_sub() { + local path="$1" + local fid + + fid=$($LFS path2fid "$path" | tr -d '[]') + [[ -n "$fid" ]] || error "cannot get FID of '$path'" + + lipe_find_expect_fid $fid true "-fid $fid" + lipe_find_expect_fid $fid true "-expr 'fid_match(\\\"$fid\\\")'" + lipe_find_expect_fid $fid true "-fid '*'" + lipe_find_expect_fid $fid true "-fid '*${fid:3}'" + lipe_find_expect_fid $fid true "-fid '${fid%:*}*'" + lipe_find_expect_fid $fid true "-fid '*${fid:3:12}*'" + lipe_find_expect_fid $fid false "-fid X$fid" + lipe_find_expect_fid $fid false "-fid ${fid}X" +} + +test_3() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_3_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_3_sub $td/$file + done +} +run_test 3 "lipe_find -fid works" + +test_4_sub() { + local file="$1" + local name=$(basename $file) + + # -name is the same with -iname except the match is case sensitive + lipe_find_expect_file $file true "-name $name" + lipe_find_expect_file $file true "-expr 'fname_match(\\\"$name\\\")'" + lipe_find_expect_file $file false "-name ${name^^}" + lipe_find_expect_file $file false "-name ${name^}" + lipe_find_expect_file $file true "-name '*'" + lipe_find_expect_file $file false "! -name $name" + lipe_find_expect_file $file true "! -name ${name^^}" + lipe_find_expect_file $file true "! -name ${name^}" + lipe_find_expect_file $file false "! -name '*'" + + # -iname is the same with -name except the match is case insensitive + lipe_find_expect_file $file true "-iname $name" + lipe_find_expect_file $file true "-expr 'fname_imatch(\\\"$name\\\")'" + lipe_find_expect_file $file true "-iname ${name^^}" + lipe_find_expect_file $file true \ + "-expr 'fname_imatch(\\\"${name^^}\\\")'" + lipe_find_expect_file $file true "-iname ${name^}" + lipe_find_expect_file $file true "-iname '*'" + lipe_find_expect_file $file false "! -iname $name" + lipe_find_expect_file $file false "! -iname ${name^^}" + lipe_find_expect_file $file false "! -iname ${name^}" + lipe_find_expect_file $file false "! -iname '*'" +} + +test_4() { + local td=$DIR/$tdir + local tf=$td/${tfile//-/_} + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_4_sub $td/$file + done +} +run_test 4 "lipe_find -name and -iname works" + +test_5_sub() { + local file="$1" + local inum + + inum=$(stat -c %i $file) + [[ -n "$inum" ]] || error "cannot get inode number of '$file'" + + lipe_find_expect_file $file true "-inum $inum" + lipe_find_expect_file $file true "-expr 'inum == $inum'" + lipe_find_expect_file $file true "-expr '== inum $inum'" + lipe_find_expect_file $file false "! -inum $inum" +} + +test_5() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + # mount Lustre clients on MDS nodes + zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT || + error "failed to mount Lustre clients on MDS nodes" + stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \ + $MOUNT" + + local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT + LIPE_FIND_USE_MOUNT=true + stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT" + + echo -e "\nTest $td" + test_5_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_5_sub $td/$file + done +} +run_test 5 "lipe_find -inum works" + +test_6_sub() { + local path="$1" + local fid + local t_option + local t_constant + local name=$(basename $path) + + fid=$($LFS path2fid "$path" | tr -d '[]') + [[ -n "$fid" ]] || error "cannot get FID of '$path'" + + t_option=$(convert_file_type option "$(stat -c %F $path)") + [[ -n "$t_option" ]] || error "cannot get type option of '$path'" + + t_constant=$(convert_file_type constant "$(stat -c %F $path)") + [[ -n "$t_constant" ]] || error "cannot get type constant of '$path'" + + lipe_find_expect_fid $fid true "-type $t_option" + lipe_find_expect_fid $fid false "! -type $t_option" + lipe_find_expect_fid $fid true "-expr '== type $t_constant'" + lipe_find_expect_fid $fid true "-expr 'type == $t_constant'" + lipe_find_expect_fid $fid true \ + "-expr '(type == $t_constant) && fid_match(\\\"$fid\\\")'" + lipe_find_expect_fid $fid true \ + "-expr '(type==$t_constant) && fid_match(\\\"$fid\\\")'" + lipe_find_expect_fid $fid true \ + "-expr '(type != $t_constant) || fid_match(\\\"$fid\\\")'" + + lipe_find_expect_fid $fid true "-name $name -type $t_option" + lipe_find_expect_fid $fid true "-name '*' -type $t_option" +} + +test_6() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_6_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_6_sub $td/$file + done +} +run_test 6 "lipe_find -type works" + +test_7_sub() { + local path="$1" + local fid + local pool + + fid=$($LFS path2fid "$path" | tr -d '[]') + [[ -n "$fid" ]] || error "cannot get FID of '$path'" + + pool=$($LFS getstripe -p "$path" | head -1) + + if [[ -z "$pool" ]]; then + lipe_find_expect_fid $fid false "-pool '*'" + lipe_find_expect_fid $fid false "-pool-regex '.*'" + return + fi + + lipe_find_expect_fid $fid true "-pool '*'" + lipe_find_expect_fid $fid true "-pool-regex '.*'" + lipe_find_expect_fid $fid true "-pool $pool" + lipe_find_expect_fid $fid true "-pool-regex $pool" + lipe_find_expect_fid $fid false "-pool ${pool}x" + lipe_find_expect_fid $fid false "-pool-regex ${pool}x" + lipe_find_expect_fid $fid false "-pool x$pool" + lipe_find_expect_fid $fid false "-pool-regex x$pool" + lipe_find_expect_fid $fid true "-expr 'pool_match(\\\"$pool\\\")'" + lipe_find_expect_fid $fid true "-expr 'pool_reg(\\\"$pool\\\")'" +} + +test_7() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_regular_files --with-pool $tf + + echo -e "\nTest $td" + test_7_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_7_sub $td/$file + done +} +run_test 7 "lipe_find -pool works" + +test_8_sub() { + local file="$1" + local type + local count + local size + local max_stripe_size + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFREG && $type != S_IFDIR ]] || + [[ "$($LFS getstripe -c $file)" =~ "no stripe info" ]]; then + lipe_find_expect_file $file false "-stripe-count 1" + lipe_find_expect_file $file false "! -stripe-count 1" + lipe_find_expect_file $file false "-stripe-count +1" + + lipe_find_expect_file $file false "-stripe-size 1048576" + lipe_find_expect_file $file false "! -stripe-size 1048576" + lipe_find_expect_file $file false "-stripe-size +1" + return + fi + + count=$($LFS getstripe -c $file) + [[ -n "$count" ]] || error "cannot get stripe count of '$file'" + + size=$($LFS getstripe -S $file) + [[ -n "$size" ]] || error "cannot get stripe size of '$file'" + + max_stripe_size=$(lipe_find_max_stripe_size $file) + + lipe_find_expect_file $file true "-stripe-count $count" + lipe_find_expect_file $file false "! -stripe-count $count" + lipe_find_expect_file $file false "-stripe-count +2000" + lipe_find_expect_file $file true "-expr 'stripe_count == $count'" + lipe_find_expect_file $file false "-expr 'stripe_count != $count'" + lipe_find_expect_file $file true "-expr 'stripe_count >= $count'" + lipe_find_expect_file $file true "-expr 'stripe_count <= $count'" + + lipe_find_expect_file $file false "-stripe-size $((size - 1))" + lipe_find_expect_file $file false "-stripe-size $((size + 1))" + lipe_find_expect_file $file false "-stripe-size +$max_stripe_size" + + if [[ $type != S_IFDIR ]]; then + lipe_find_expect_file $file true "-expr 'stripe_size == $size'" + lipe_find_expect_file $file false "-expr 'stripe_size != $size'" + lipe_find_expect_file $file true "-expr 'stripe_size >= $size'" + lipe_find_expect_file $file true "-expr 'stripe_size <= $size'" + fi +} + +test_8() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + $LFS setstripe -c 1 $td || error "$LFS setstripe -c 1 $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_8_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_8_sub $td/$file + done +} +run_test 8 "lipe_find -stripe-count and -stripe-size work" + +test_9_sub() { + local file="$1" + local type + local idx + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFREG && $type != S_IFDIR ]] || + [[ "$($LFS getstripe -i $file)" =~ "no stripe info" ]]; then + lipe_find_expect_file $file false "-ost 0" + lipe_find_expect_file $file false "! -ost 0" + + lipe_find_expect_file $file false "-stripe-index 0" + lipe_find_expect_file $file false "! -stripe-index 0" + return + fi + + idx=$($LFS getstripe -i $file) + [[ -n "$idx" ]] || error "cannot get stripe index of '$file'" + + if [[ $type = S_IFDIR || "$($LFS getstripe -L $file)" == "mdt" ]]; then + lipe_find_expect_file $file false "-ost $idx" + lipe_find_expect_file $file true "! -ost $idx" + lipe_find_expect_file $file false "-ost $((idx + 1))" + lipe_find_expect_file $file true "! -ost $((idx + 1))" + + lipe_find_expect_file $file false "-stripe-index $idx" + lipe_find_expect_file $file true "! -stripe-index $idx" + lipe_find_expect_file $file false "-stripe-index $((idx + 1))" + lipe_find_expect_file $file true "! -stripe-index $((idx + 1))" + elif [[ $type = S_IFREG ]]; then + lipe_find_expect_file $file true "-ost $idx" + lipe_find_expect_file $file true "-expr 'ost($idx)'" + lipe_find_expect_file $file false "! -ost $idx" + lipe_find_expect_file $file false "-ost $((idx + OSTCOUNT))" + lipe_find_expect_file $file true "! -ost $((idx + OSTCOUNT))" + + lipe_find_expect_file $file true "-stripe-index $idx" + lipe_find_expect_file $file false "! -stripe-index $idx" + lipe_find_expect_file $file false \ + "-stripe-index $((idx + OSTCOUNT))" + lipe_find_expect_file $file true \ + "! -stripe-index $((idx + OSTCOUNT))" + fi +} + +test_9() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + $LFS setstripe -i 0 $td || error "$LFS setstripe -i 0 $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_9_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_9_sub $td/$file + done +} +run_test 9 "lipe_find -ost and -stripe-index work" + +test_10_sub() { + local file="$1" + local type + local count + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFREG && $type != S_IFDIR ]] || + [[ "$($LFS getstripe --comp-count $file)" =~ "no stripe info" ]] + then + lipe_find_expect_file $file false "-comp-count 0" + lipe_find_expect_file $file false "! -comp-count 0" + lipe_find_expect_file $file false "-comp-count +1" + lipe_find_expect_file $file false "-comp-count -1" + + lipe_find_expect_file $file false "-component-count 0" + lipe_find_expect_file $file false "! -component-count 0" + lipe_find_expect_file $file false "-component-count +1" + lipe_find_expect_file $file false "-component-count -1" + return + fi + + count=$($LFS getstripe --comp-count $file) + [[ -n "$count" ]] || error "cannot get component count of '$file'" + (( count != 0 )) || return 0 + + lipe_find_expect_file $file true "-comp-count $count" + lipe_find_expect_file $file false "! -comp-count $count" + lipe_find_expect_file $file false "-comp-count +$count" + lipe_find_expect_file $file false "-comp-count -$count" + lipe_find_expect_file $file true "-comp-count -$((count + 1))" + + lipe_find_expect_file $file true "-expr 'comp_count == $count'" + lipe_find_expect_file $file false "-expr 'comp_count != $count'" + lipe_find_expect_file $file true "-expr 'comp_count >= $count'" + lipe_find_expect_file $file true "-expr 'comp_count <= $count'" + + lipe_find_expect_file $file true "-component-count $count" + lipe_find_expect_file $file false "! -component-count $count" + lipe_find_expect_file $file false "-component-count $((count + 1))" + lipe_find_expect_file $file true "! -component-count $((count + 1))" + lipe_find_expect_file $file true "-component-count +$((count - 1))" +} + +test_10() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_10_sub $td/$file + done +} +run_test 10 "lipe_find -comp-count and -component-count work" + +test_11_sub() { + local file="$1" + local type + local layout + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFREG || + "$($LFS getstripe -L $file)" =~ "no stripe info" ]]; then + lipe_find_expect_file $file false "-layout raid0" + lipe_find_expect_file $file false "-layout mdt" + lipe_find_expect_file $file false "-layout released" + + lipe_find_expect_file $file false "! -layout raid0" + lipe_find_expect_file $file false "! -layout mdt" + lipe_find_expect_file $file false "! -layout released" + return + fi + + layout=$($LFS getstripe -L $file) + [[ -n "$layout" ]] || error "cannot get file layout of '$file'" + + lipe_find_expect_file $file false "-layout released" + lipe_find_expect_file $file true "! -layout released" + lipe_find_expect_file $file true "-layout $layout" + lipe_find_expect_file $file false "! -layout $layout" + lipe_find_expect_file $file true "-layout $layout,$layout" + lipe_find_expect_file $file false "-layout $layout,released" + lipe_find_expect_file $file false "-layout raid0,mdt,released" + lipe_find_expect_file $file true "-expr 'layout(\\\"$layout\\\")'" + lipe_find_expect_file $file false \ + "-expr 'layout(\\\"mdt,raid0,released\\\")'" +} + +test_11() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_11_sub $td/$file + done +} +run_test 11 "lipe_find -layout works" + +test_12_sub() { + local file="$1" + local uid + local user + + uid=$(stat -c %u $file) + [[ -n "$uid" ]] || error "cannot get user ID of owner for '$file'" + + user=$(stat -c %U $file) + [[ -n "$user" ]] || error "cannot get user name of owner for '$file'" + + lipe_find_expect_file $file true "-uid $uid" + lipe_find_expect_file $file true "! ! -uid $uid" + lipe_find_expect_file $file false "! -uid $uid" + lipe_find_expect_file $file false "! -uid $((uid + 1)) ! -uid $uid" + lipe_find_expect_file $file true "-expr 'uid == $uid'" + lipe_find_expect_file $file false "-expr 'uid != $uid'" + lipe_find_expect_file $file true \ + "-expr 'uid == user2id(\\\"$user\\\")'" + + lipe_find_expect_file $file true "-user $uid" + lipe_find_expect_file $file true "! ! -user $uid" + lipe_find_expect_file $file false "! -user $uid" + lipe_find_expect_file $file false "! -user $((uid + 1)) ! -user $uid" + lipe_find_expect_file $file true "-expr 'user(\\\"$uid\\\")'" + + lipe_find_expect_file $file true "-user $user" + lipe_find_expect_file $file true "! ! -user $user" + lipe_find_expect_file $file false "! -user $user" + lipe_find_expect_file $file false "! -user bin ! -user $user" + lipe_find_expect_file $file false "-nouser" + lipe_find_expect_file $file false "-expr nouser" + lipe_find_expect_file $file true "-expr 'user(\\\"$user\\\")'" + + [[ $uid != $RUNAS_ID ]] || return 0 + + chown $RUNAS_ID $file || return 0 + + lipe_find_expect_file $file false "-uid $uid" + lipe_find_expect_file $file true "! -uid $uid" + lipe_find_expect_file $file true "-uid $RUNAS_ID" + lipe_find_expect_file $file false "! -uid $RUNAS_ID" +} + +test_12() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_12_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_12_sub $td/$file + done +} +run_test 12 "lipe_find -uid, -user and -nouser work" + +test_13_sub() { + local file="$1" + local gid + local group + + gid=$(stat -c %g $file) + [[ -n "$gid" ]] || error "cannot get group ID of owner for '$file'" + + group=$(stat -c %G $file) + [[ -n "$group" ]] || error "cannot get group name of owner for '$file'" + + lipe_find_expect_file $file true "-gid $gid" + lipe_find_expect_file $file true "! ! -gid $gid" + lipe_find_expect_file $file false "! -gid $gid" + lipe_find_expect_file $file false "! -gid $((gid + 1)) ! -gid $gid" + lipe_find_expect_file $file true "-expr 'gid == $gid'" + lipe_find_expect_file $file false "-expr 'gid != $gid'" + lipe_find_expect_file $file true \ + "-expr 'gid == group2id(\\\"$group\\\")'" + + lipe_find_expect_file $file true "-group $gid" + lipe_find_expect_file $file true "! ! -group $gid" + lipe_find_expect_file $file false "! -group $gid" + lipe_find_expect_file $file false "! -group $((gid + 1)) ! -group $gid" + lipe_find_expect_file $file true "-expr 'group(\\\"$gid\\\")'" + + lipe_find_expect_file $file true "-group $group" + lipe_find_expect_file $file true "! ! -group $group" + lipe_find_expect_file $file false "! -group $group" + lipe_find_expect_file $file false "! -group bin ! -group $group" + lipe_find_expect_file $file false "-nogroup" + lipe_find_expect_file $file false "-expr nogroup" + lipe_find_expect_file $file true "-expr 'group(\\\"$group\\\")'" + + [[ $gid != $RUNAS_GID ]] || return 0 + + chown :$RUNAS_GID $file || return 0 + + lipe_find_expect_file $file false "-gid $gid" + lipe_find_expect_file $file true "! -gid $gid" + lipe_find_expect_file $file true "-gid $RUNAS_GID" + lipe_find_expect_file $file false "! -gid $RUNAS_GID" +} + +test_13() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_13_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_13_sub $td/$file + done +} +run_test 13 "lipe_find -gid, -group and -nogroup work" + +test_14_sub() { + local file="$1" + local uid + local gid + local xtime + local expr_cmd + + uid=$(stat -c %u $file) + [[ -n "$uid" ]] || error "cannot get user ID of owner for '$file'" + + gid=$(stat -c %g $file) + [[ -n "$gid" ]] || error "cannot get group ID of owner for '$file'" + + for xtime in atime ctime mtime; do + lipe_find_expect_file $file true "-$xtime 0" + lipe_find_expect_file $file false "! -$xtime 0" + lipe_find_expect_file $file false "-$xtime +0" + lipe_find_expect_file $file true "! -$xtime +0" + lipe_find_expect_file $file false "-$xtime -0" + lipe_find_expect_file $file true "! -$xtime -0" + lipe_find_expect_file $file true "-$xtime -1" + lipe_find_expect_file $file false "! -$xtime -1" + lipe_find_expect_file $file true "-expr '$xtime > 0'" + lipe_find_expect_file $file false "-expr '$xtime < 0'" + + expr_cmd="-expr '($xtime > (sys_time - 1 * days) ||" + expr_cmd+=" uid == $((uid + 100))) && gid == $gid'" + lipe_find_expect_file $file true "$expr_cmd" + done +} + +test_14() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_14_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_14_sub $td/$file + done +} +run_test 14 "lipe_find -atime, -ctime and -mtime work" + +test_15_sub() { + local file="$1" + local xmin + + for xmin in amin cmin mmin; do + lipe_find_expect_file $file false "-$xmin +600" + lipe_find_expect_file $file true "! -$xmin +600" + lipe_find_expect_file $file false "-$xmin -0" + lipe_find_expect_file $file true "! -$xmin -0" + lipe_find_expect_file $file true "-$xmin -600" + lipe_find_expect_file $file false "! -$xmin -600" + done +} + +test_15() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_15_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_15_sub $td/$file + done +} +run_test 15 "lipe_find -amin, -cmin and -mmin work" + +test_16_sub() { + local file="$1" + + lipe_find_expect_file $file false "-used +0" + lipe_find_expect_file $file false "-used 1" + lipe_find_expect_file $file true "-used -1" + lipe_find_expect_file $file false "-used +1" +} + +test_16() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_16_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_16_sub $td/$file + done +} +run_test 16 "lipe_find -used works" + +test_17_sub() { + local file="$1" + local type + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFREG && $type != S_IFDIR ]]; then + lipe_find_expect_file $file false "-empty" + lipe_find_expect_file $file true "! -empty" + return + fi + + if [[ $type = S_IFDIR ]]; then + if [[ -z "$(ls $file)" ]]; then + lipe_find_expect_file $file true "-empty" + lipe_find_expect_file $file false "! -empty" + else + lipe_find_expect_file $file false "-empty" + lipe_find_expect_file $file true "! -empty" + fi + return + fi + + # S_IFREG + if (( $(stat -c %s $file) == 0 )); then + lipe_find_expect_file $file true "-empty" + lipe_find_expect_file $file false "! -empty" + echo foo > $file || error "echo foo > $file failed" + sync; sleep 5; sync + fi + + lipe_find_expect_file $file false "-empty" + lipe_find_expect_file $file true "! -empty" + lipe_find_expect_file $file false "-expr empty" +} + +test_17() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + mkdir $td-1 || error "mkdir $td-1 failed" + create_lipe_find_files $tf + + # mount Lustre clients on MDS nodes + zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT || + error "failed to mount Lustre clients on MDS nodes" + stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \ + $MOUNT" + + local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT + LIPE_FIND_USE_MOUNT=true + stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT" + + echo -e "\nTest $td" + test_17_sub $td + + echo -e "\nTest $td-1" + test_17_sub $td-1 + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_17_sub $td/$file + done +} +run_test 17 "lipe_find -empty works" + +test_18_sub() { + local file="$1" + local type + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + if [[ $type != S_IFDIR ]]; then + lipe_find_expect_file $file true "-entries 0" + lipe_find_expect_file $file false "! -entries 0" + lipe_find_expect_file $file true "-expr 'entries == 0'" + lipe_find_expect_file $file false "-expr 'entries != 0'" + return + fi + + if [[ -z "$(ls $file)" ]]; then + lipe_find_expect_file $file true "-entries 0" + lipe_find_expect_file $file false "! -entries 0" + lipe_find_expect_file $file false "-entries +0" + lipe_find_expect_file $file true "! -entries +0" + lipe_find_expect_file $file false "-entries -0" + lipe_find_expect_file $file true "! -entries -0" + else + lipe_find_expect_file $file false "-entries 0" + lipe_find_expect_file $file true "! -entries 0" + lipe_find_expect_file $file true "-entries +0" + lipe_find_expect_file $file false "! -entries +0" + lipe_find_expect_file $file false "-entries -0" + lipe_find_expect_file $file true "! -entries -0" + fi +} + +test_18() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + mkdir $td-1 || error "mkdir $td-1 failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_18_sub $td + + echo -e "\nTest $td-1" + test_18_sub $td-1 + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_18_sub $td/$file + done +} +run_test 18 "lipe_find -entries works" + +test_19_sub() { + local file="$1" + local nlink + + nlink=$(stat -c %h $file) + [[ -n "$nlink" ]] || error "cannot get number of hard links for '$file'" + + lipe_find_expect_file $file true "-links $nlink" + lipe_find_expect_file $file false "! -links $nlink" + lipe_find_expect_file $file false "-links $((nlink + 1))" + lipe_find_expect_file $file false "-links $((nlink - 1))" + lipe_find_expect_file $file true "-expr 'nlink == $nlink'" + lipe_find_expect_file $file false "-expr 'nlink != $nlink'" + lipe_find_expect_file $file true "-expr 'nlink >= $nlink'" + lipe_find_expect_file $file true "-expr 'nlink <= $nlink'" +} + +test_19() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_19_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_19_sub $td/$file + done +} +run_test 19 "lipe_find -links works" + +test_20_sub() { + local file="$1" + local type + local cmd + + type=$(convert_file_type constant "$(stat -c %F $file)") + [[ -n "$type" ]] || error "cannot get type of '$file'" + + [[ $type = S_IFREG || $type = S_IFDIR ]] || return 0 + + cmd="setfattr -h -n user.xattr_name -v xattr_value $file" + echo $cmd + $cmd || error "setfattr $file failed" + + lipe_find_expect_file $file true "-xattr 'user.xattr_name=*'" + lipe_find_expect_file $file true "-xattr '*=xattr_value'" + lipe_find_expect_file $file true "-xattr 'user.xattr_name=xattr_value'" + lipe_find_expect_file $file true "-xattr 'user.xattr_name*=xattr_value'" + lipe_find_expect_file $file true "-xattr 'user.*=xattr_value'" + lipe_find_expect_file $file true "-xattr 'user.*=*_value'" + lipe_find_expect_file $file true "-xattr '*=*'" + lipe_find_expect_file $file true "-xattr 'trusted.lma=*'" + lipe_find_expect_file $file false "-xattr 'invalid_name=*'" + lipe_find_expect_file $file false "-xattr '*=invalid_value'" + lipe_find_expect_file $file true "-expr \ + 'xattr_match(\\\"user.xattr_name\\\", \\\"xattr_value\\\")'" +} + +test_20() { + which setfattr &>/dev/null || skip_env "setfattr is not installed" + + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_20_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_20_sub $td/$file + done +} +run_test 20 "lipe_find -xattr works" + +test_21_sub() { + local file="$1" + local perm + + perm=$(stat -c %a $file) + [[ -n "$perm" ]] || error "cannot get access rights for '$file'" + + lipe_find_expect_file $file true "-perm -0" + lipe_find_expect_file $file true "-perm /0" + + lipe_find_expect_file $file true "-perm $perm" + lipe_find_expect_file $file true "-perm -$perm" + lipe_find_expect_file $file true "-perm /$perm" + + chmod 666 $file || return 0 + + lipe_find_expect_file $file false "-perm 777" + lipe_find_expect_file $file false "-perm -777" + lipe_find_expect_file $file true "-perm /777" + + lipe_find_expect_file $file false "-expr 'perm_equal(444)'" + lipe_find_expect_file $file true "-expr 'perm_at_least(444)'" + lipe_find_expect_file $file true "-expr 'perm_any(444)'" +} + +test_21() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + echo -e "\nTest $td" + test_21_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_21_sub $td/$file + done +} +run_test 21 "lipe_find -perm works" + +test_22_sub() { + local file="$1" + + lipe_find_expect_file $file true "-projid 0" + lipe_find_expect_file $file false "! -projid 0" + lipe_find_expect_file $file true "-expr 'projid == 0'" + lipe_find_expect_file $file true "-expr '== projid 0'" + lipe_find_expect_file $file false "-projid +0" + lipe_find_expect_file $file true "-projid -1" +} + +test_22() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + # mount Lustre clients on MDS nodes + zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT || + error "failed to mount Lustre clients on MDS nodes" + stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \ + $MOUNT" + + local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT + LIPE_FIND_USE_MOUNT=true + stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT" + + echo -e "\nTest $td" + test_22_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_22_sub $td/$file + done +} +run_test 22 "lipe_find -projid works" + +test_23_sub() { + local file="$1" + local size + local f_size + local b_num + local b_size + local b_usize + local opt + + f_size=$(stat -c %s $file) + [[ -n "$f_size" ]] || error "cannot get total size for '$file'" + + b_num=$(stat -c %b $file) + [[ -n "$b_num" ]] || error "cannot get number of blocks for '$file'" + + b_usize=$(stat -c %B $file) + [[ -n "$b_usize" ]] || error "cannot get block unit size for '$file'" + + b_size=$((b_num * b_usize)) + + for opt in size blocks; do + [[ $opt = size ]] && size=$f_size || size=$b_size + + lipe_find_expect_file $file true "-$opt ${size}c" + lipe_find_expect_file $file false "! -$opt ${size}c" + lipe_find_expect_file $file false "-$opt +${size}c" + lipe_find_expect_file $file false "-$opt -${size}c" + + lipe_find_expect_file $file false "-$opt $((size + 1))c" + lipe_find_expect_file $file true "! -$opt $((size + 1))c" + lipe_find_expect_file $file false "-$opt +$((size + 1))c" + lipe_find_expect_file $file true "-$opt -$((size + 1))c" + + (( size != 0 )) || continue + + lipe_find_expect_file $file true "-$opt $((size / 512))b" + lipe_find_expect_file $file true "-$opt $((size / 1024))" + lipe_find_expect_file $file true "-$opt $((size / 1048576))M" + + lipe_find_expect_file $file true "-expr '$opt == $size'" + lipe_find_expect_file $file true "-expr '$opt >= $size'" + lipe_find_expect_file $file true "-expr '$opt <= $size'" + lipe_find_expect_file $file false "-expr '$opt != $size'" + + lipe_find_expect_file $file false "-$opt $((size - 1))c" + lipe_find_expect_file $file true "! -$opt $((size - 1))c" + lipe_find_expect_file $file true "-$opt +$((size - 1))c" + lipe_find_expect_file $file false "-$opt -$((size - 1))c" + done +} + +test_23() { + local td=$DIR/$tdir + local tf=$td/$tfile + local file + + mkdir $td || error "mkdir $td failed" + create_lipe_find_files $tf + + # mount Lustre clients on MDS nodes + zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT || + error "failed to mount Lustre clients on MDS nodes" + stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \ + $MOUNT" + + local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT + LIPE_FIND_USE_MOUNT=true + stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT" + + echo -e "\nTest $td" + test_23_sub $td + + for file in $(ls $td); do + echo -e "\nTest $td/$file" + test_23_sub $td/$file + done +} +run_test 23 "lipe_find -size and -blocks work" + +complete $SECONDS +check_and_cleanup_lustre +exit_status diff --git a/lustre/tests/test-groups/regression b/lustre/tests/test-groups/regression index b4a6da6..c88e737 100644 --- a/lustre/tests/test-groups/regression +++ b/lustre/tests/test-groups/regression @@ -27,3 +27,4 @@ sanity-lsnapshot sanity-pfl sanity-pcc sanity-lnet +sanity-lipe -- 1.8.3.1