LU-14824 test: sanity 413a/b unlink timeout

Unlinking remote/striped directories is slow on zfs system, limit
total directory number for 1-stripe directory test in 413a/b on zfs
system, and don't test striped directory to avoid timeout.

Also limit total stripe object count to avoid timeout.

Test-Parameters: trivial
Test-Parameters: mdscount=2 mdtcount=4 fstype=ldiskfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50
Test-Parameters: mdscount=2 mdtcount=4 fstype=zfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50
Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: Ie116e6df5aee3877ed9f093f58e7bd71f6c6d9d5
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/45955
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Arshad Hussain <arshad.hussain@aeoncomputing.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index cbe3909..687d8fd 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -26070,10 +26070,11 @@ generate_uneven_mdts() {
                fi
        done
 
+       (( min > 0 )) || skip "low space on MDT$min_index"
        (( ${ffree[min_index]} > 0 )) ||
-               skip "no free files in MDT$min_index"
+               skip "no free files on MDT$min_index"
        (( ${ffree[min_index]} < 10000000 )) ||
-               skip "too many free files in MDT$min_index"
+               skip "too many free files on MDT$min_index"
 
        # Check if we need to generate uneven MDTs
        local diff=$(((max - min) * 100 / min))
@@ -26108,6 +26109,8 @@ generate_uneven_mdts() {
                        (${bavail[max_index]} * bsize >> 16)))
                min=$(((${ffree[min_index]} >> 8) *
                        (${bavail[min_index]} * bsize >> 16)))
+               (( min > 0 )) ||
+                       skip "low space on MDT$min_index"
                diff=$(((max - min) * 100 / min))
                i=$((i + 1))
        done
@@ -26129,9 +26132,17 @@ test_qos_mkdir() {
        local lod_qos_prio_free
        local lod_qos_threshold_rr
        local lod_qos_maxage
+       local total
        local count
        local i
 
+       # @total is total directories created if it's testing plain
+       # directories, otherwise it's total stripe object count for
+       # striped directories test.
+       # remote/striped directory unlinking is slow on zfs and may
+       # timeout, test with fewer directories
+       [ "$mds1_FSTYPE" = "zfs" ] && total=120 || total=240
+
        lmv_qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1)
        lmv_qos_prio_free=${lmv_qos_prio_free%%%}
        lmv_qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr |
@@ -26180,8 +26191,8 @@ test_qos_mkdir() {
                echo "Mkdir (stripe_count $stripe_count) roundrobin:" ||
                echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index"
 
-       stack_trap "unlinkmany -d $testdir/subdir $((100 * MDSCOUNT))"
-       for (( i = 0; i < 100 * MDSCOUNT; i++ )); do
+       stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))"
+       for (( i = 0; i < total / stripe_count; i++ )); do
                eval $mkdir_cmd $testdir/subdir$i ||
                        error "$mkdir_cmd subdir$i failed"
        done
@@ -26190,13 +26201,13 @@ test_qos_mkdir() {
                count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$")
                echo "$count directories created on MDT$i"
                if $test_mkdir_rr; then
-                       (( $count == 100 )) ||
+                       (( count == total / stripe_count / MDSCOUNT )) ||
                                error "subdirs are not evenly distributed"
-               elif (( $i == $stripe_index )); then
-                       (( $count == 100 * MDSCOUNT )) ||
+               elif (( i == stripe_index )); then
+                       (( count == total / stripe_count )) ||
                                error "$count subdirs created on MDT$i"
                else
-                       (( $count == 0 )) ||
+                       (( count == 0 )) ||
                                error "$count subdirs created on MDT$i"
                fi
 
@@ -26205,8 +26216,9 @@ test_qos_mkdir() {
                                grep -c -P "^\s+$i\t")
                        echo "$count stripes created on MDT$i"
                        # deviation should < 5% of average
-                       (( $count >= 95 * stripe_count &&
-                          $count <= 105 * stripe_count)) ||
+                       delta=$((count - total / MDSCOUNT))
+                       delta=${delta#-}
+                       (( delta <= total / MDSCOUNT / 20 )) ||
                                error "stripes are not evenly distributed"
                fi
        done
@@ -26242,10 +26254,12 @@ test_qos_mkdir() {
                fi
        done
 
-       (( ${ffree[min_index]} > 0 )) ||
-               skip "no free files in MDT$min_index"
+       (( min > 0 )) ||
+               skip "low space on MDT$min_index"
        (( ${ffree[min_index]} < 10000000 )) ||
-               skip "too many free files in MDT$min_index"
+               skip "too many free files on MDT$min_index"
+
+       generate_uneven_mdts 120
 
        echo "MDT filesfree available: ${ffree[*]}"
        echo "MDT blocks available: ${bavail[*]}"
@@ -26264,10 +26278,9 @@ test_qos_mkdir() {
        sleep 1
 
        testdir=$DIR/$tdir-s$stripe_count/qos
-       local num=200
 
-       stack_trap "unlinkmany -d $testdir/subdir $((num * MDSCOUNT))"
-       for (( i = 0; i < num * MDSCOUNT; i++ )); do
+       stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))"
+       for (( i = 0; i < total / stripe_count; i++ )); do
                eval $mkdir_cmd $testdir/subdir$i ||
                        error "$mkdir_cmd subdir$i failed"
        done
@@ -26282,8 +26295,8 @@ test_qos_mkdir() {
        min=$($LFS getdirstripe -i $testdir/* | grep -c "^$min_index$")
 
        # D-value should > 10% of averge
-       (( max - min > num / 10 )) ||
-               error "subdirs shouldn't be evenly distributed: $max - $min < $((num / 10))"
+       (( max - min > total / stripe_count / MDSCOUNT / 10 )) ||
+               error "subdirs shouldn't be evenly distributed: $max - $min <= $((total / stripe_count / MDSCOUNT / 10))"
 
        # ditto for stripes
        if (( stripe_count > 1 )); then
@@ -26297,8 +26310,8 @@ test_qos_mkdir() {
 
                min=$($LFS getdirstripe $testdir/* |
                        grep -c -P "^\s+$min_index\t")
-               (( max - min > num * stripe_count / 10 )) ||
-                       error "stripes shouldn't be evenly distributed: $max - $min < $((num / 10)) * $stripe_count"
+               (( max - min > total / MDSCOUNT / 10 )) ||
+                       error "stripes shouldn't be evenly distributed: $max - $min <= $((total / MDSCOUNT / 10))"
        fi
 }
 
@@ -26331,10 +26344,17 @@ test_413a() {
        [ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
                skip "Need server version at least 2.12.52"
 
+       local stripe_max=$((MDSCOUNT - 1))
        local stripe_count
 
-       generate_uneven_mdts 100
-       for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+       generate_uneven_mdts 120
+
+       # test 4-stripe directory at most, otherwise it's too slow
+       (( stripe_max > 4 )) && stripe_max=4
+       # unlinking striped directory is slow on zfs, and may timeout, only test
+       # plain directory
+       [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1
+       for stripe_count in $(seq 1 $stripe_max); do
                mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed"
                mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed"
                $LFS mkdir -i $(most_full_mdt) $DIR/$tdir-s$stripe_count/qos ||
@@ -26351,11 +26371,15 @@ test_413b() {
        [ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
                skip "Need server version at least 2.12.52"
 
+       local stripe_max=$((MDSCOUNT - 1))
        local testdir
        local stripe_count
 
-       generate_uneven_mdts 100
-       for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+       generate_uneven_mdts 120
+
+       (( stripe_max > 4 )) && stripe_max=4
+       [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1
+       for stripe_count in $(seq 1 $stripe_max); do
                testdir=$DIR/$tdir-s$stripe_count
                mkdir $testdir || error "mkdir $testdir failed"
                mkdir $testdir/rr || error "mkdir rr failed"
@@ -26381,6 +26405,8 @@ test_413c() {
        local inherit
        local inherit_rr
 
+       generate_uneven_mdts 120
+
        testdir=$DIR/${tdir}-s1
        mkdir $testdir || error "mkdir $testdir failed"
        mkdir $testdir/rr || error "mkdir rr failed"
@@ -26618,6 +26644,8 @@ test_413z() {
        for pid in $pids; do
                wait $pid
        done
+
+       true
 }
 run_test 413z "413 test cleanup"