From 5ff3e400f1a74ea49b7eb9cf19715f0fae08c3f5 Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Wed, 22 Dec 2021 07:26:49 -0500 Subject: [PATCH] LU-14824 test: sanity 413a/b unlink timeout Unlinking remote/striped directories is slow on zfs system, limit total directory number for 1-stripe directory test in 413a/b on zfs system, and don't test striped directory to avoid timeout. Also limit total stripe object count to avoid timeout. Test-Parameters: trivial Test-Parameters: mdscount=2 mdtcount=4 fstype=ldiskfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50 Test-Parameters: mdscount=2 mdtcount=4 fstype=zfs testlist=sanity env=ONLY="413a 413b",ONLY_REPEAT=50 Signed-off-by: Lai Siyao Change-Id: Ie116e6df5aee3877ed9f093f58e7bd71f6c6d9d5 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/45955 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Arshad Hussain --- lustre/tests/sanity.sh | 76 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index cbe3909..687d8fd 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -26070,10 +26070,11 @@ generate_uneven_mdts() { fi done + (( min > 0 )) || skip "low space on MDT$min_index" (( ${ffree[min_index]} > 0 )) || - skip "no free files in MDT$min_index" + skip "no free files on MDT$min_index" (( ${ffree[min_index]} < 10000000 )) || - skip "too many free files in MDT$min_index" + skip "too many free files on MDT$min_index" # Check if we need to generate uneven MDTs local diff=$(((max - min) * 100 / min)) @@ -26108,6 +26109,8 @@ generate_uneven_mdts() { (${bavail[max_index]} * bsize >> 16))) min=$(((${ffree[min_index]} >> 8) * (${bavail[min_index]} * bsize >> 16))) + (( min > 0 )) || + skip "low space on MDT$min_index" diff=$(((max - min) * 100 / min)) i=$((i + 1)) done @@ -26129,9 +26132,17 @@ test_qos_mkdir() { local lod_qos_prio_free local lod_qos_threshold_rr local lod_qos_maxage + local total local count local i + # @total is total directories created if it's testing plain + # directories, otherwise it's total stripe object count for + # striped directories test. + # remote/striped directory unlinking is slow on zfs and may + # timeout, test with fewer directories + [ "$mds1_FSTYPE" = "zfs" ] && total=120 || total=240 + lmv_qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1) lmv_qos_prio_free=${lmv_qos_prio_free%%%} lmv_qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr | @@ -26180,8 +26191,8 @@ test_qos_mkdir() { echo "Mkdir (stripe_count $stripe_count) roundrobin:" || echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index" - stack_trap "unlinkmany -d $testdir/subdir $((100 * MDSCOUNT))" - for (( i = 0; i < 100 * MDSCOUNT; i++ )); do + stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))" + for (( i = 0; i < total / stripe_count; i++ )); do eval $mkdir_cmd $testdir/subdir$i || error "$mkdir_cmd subdir$i failed" done @@ -26190,13 +26201,13 @@ test_qos_mkdir() { count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$") echo "$count directories created on MDT$i" if $test_mkdir_rr; then - (( $count == 100 )) || + (( count == total / stripe_count / MDSCOUNT )) || error "subdirs are not evenly distributed" - elif (( $i == $stripe_index )); then - (( $count == 100 * MDSCOUNT )) || + elif (( i == stripe_index )); then + (( count == total / stripe_count )) || error "$count subdirs created on MDT$i" else - (( $count == 0 )) || + (( count == 0 )) || error "$count subdirs created on MDT$i" fi @@ -26205,8 +26216,9 @@ test_qos_mkdir() { grep -c -P "^\s+$i\t") echo "$count stripes created on MDT$i" # deviation should < 5% of average - (( $count >= 95 * stripe_count && - $count <= 105 * stripe_count)) || + delta=$((count - total / MDSCOUNT)) + delta=${delta#-} + (( delta <= total / MDSCOUNT / 20 )) || error "stripes are not evenly distributed" fi done @@ -26242,10 +26254,12 @@ test_qos_mkdir() { fi done - (( ${ffree[min_index]} > 0 )) || - skip "no free files in MDT$min_index" + (( min > 0 )) || + skip "low space on MDT$min_index" (( ${ffree[min_index]} < 10000000 )) || - skip "too many free files in MDT$min_index" + skip "too many free files on MDT$min_index" + + generate_uneven_mdts 120 echo "MDT filesfree available: ${ffree[*]}" echo "MDT blocks available: ${bavail[*]}" @@ -26264,10 +26278,9 @@ test_qos_mkdir() { sleep 1 testdir=$DIR/$tdir-s$stripe_count/qos - local num=200 - stack_trap "unlinkmany -d $testdir/subdir $((num * MDSCOUNT))" - for (( i = 0; i < num * MDSCOUNT; i++ )); do + stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))" + for (( i = 0; i < total / stripe_count; i++ )); do eval $mkdir_cmd $testdir/subdir$i || error "$mkdir_cmd subdir$i failed" done @@ -26282,8 +26295,8 @@ test_qos_mkdir() { min=$($LFS getdirstripe -i $testdir/* | grep -c "^$min_index$") # D-value should > 10% of averge - (( max - min > num / 10 )) || - error "subdirs shouldn't be evenly distributed: $max - $min < $((num / 10))" + (( max - min > total / stripe_count / MDSCOUNT / 10 )) || + error "subdirs shouldn't be evenly distributed: $max - $min <= $((total / stripe_count / MDSCOUNT / 10))" # ditto for stripes if (( stripe_count > 1 )); then @@ -26297,8 +26310,8 @@ test_qos_mkdir() { min=$($LFS getdirstripe $testdir/* | grep -c -P "^\s+$min_index\t") - (( max - min > num * stripe_count / 10 )) || - error "stripes shouldn't be evenly distributed: $max - $min < $((num / 10)) * $stripe_count" + (( max - min > total / MDSCOUNT / 10 )) || + error "stripes shouldn't be evenly distributed: $max - $min <= $((total / MDSCOUNT / 10))" fi } @@ -26331,10 +26344,17 @@ test_413a() { [ $MDS1_VERSION -lt $(version_code 2.12.52) ] && skip "Need server version at least 2.12.52" + local stripe_max=$((MDSCOUNT - 1)) local stripe_count - generate_uneven_mdts 100 - for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do + generate_uneven_mdts 120 + + # test 4-stripe directory at most, otherwise it's too slow + (( stripe_max > 4 )) && stripe_max=4 + # unlinking striped directory is slow on zfs, and may timeout, only test + # plain directory + [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1 + for stripe_count in $(seq 1 $stripe_max); do mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed" mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed" $LFS mkdir -i $(most_full_mdt) $DIR/$tdir-s$stripe_count/qos || @@ -26351,11 +26371,15 @@ test_413b() { [ $MDS1_VERSION -lt $(version_code 2.12.52) ] && skip "Need server version at least 2.12.52" + local stripe_max=$((MDSCOUNT - 1)) local testdir local stripe_count - generate_uneven_mdts 100 - for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do + generate_uneven_mdts 120 + + (( stripe_max > 4 )) && stripe_max=4 + [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1 + for stripe_count in $(seq 1 $stripe_max); do testdir=$DIR/$tdir-s$stripe_count mkdir $testdir || error "mkdir $testdir failed" mkdir $testdir/rr || error "mkdir rr failed" @@ -26381,6 +26405,8 @@ test_413c() { local inherit local inherit_rr + generate_uneven_mdts 120 + testdir=$DIR/${tdir}-s1 mkdir $testdir || error "mkdir $testdir failed" mkdir $testdir/rr || error "mkdir rr failed" @@ -26618,6 +26644,8 @@ test_413z() { for pid in $pids; do wait $pid done + + true } run_test 413z "413 test cleanup" -- 1.8.3.1