LU-12066 tests: activate OSTs in recovery-small/26b

In recovery-small.sh test_26b ensure that all OST connections are
active and not disconnected due to being idle.  Otherwise, the
number of OST exports for the remaining client may go down while
the test is running and they will never reconnect.

Change the wait condition to consider success if more exports have
disconnected than expected.

Move wait_client_evicted() down to test_26b() since it is the only
test case where this helper function is being used.

Minor test script style improvements.

Fixes: 5a6ceb664f07 ("LU-7236 ptlrpc: idle connections can disconnect")
Test-Parameters: trivial testlist=recovery-small env=ONLY=26b,ONLY_REPEAT=100
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I7b21d89ea148c4fda6527870e491f59abb3ebbe5
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/56356
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Arshad Hussain <arshad.hussain@aeoncomputing.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 8881494..64856f4 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -1022,16 +1022,6 @@ test_24a() { # bug 11710 details correct fsync() behavior
 }
 run_test 24a "fsync error (should return error)"
 
-wait_client_evicted () {
-       local facet=$1
-       local exports=$2
-       local varsvc=${facet}_svc
-
-       wait_update $(facet_active_host $facet) \
-               "lctl get_param -n *.${!varsvc}.num_exports | cut -d' ' -f2" \
-               $((exports - 1)) $3
-}
-
 test_24b() {
        remote_ost_nodsh && skip "remote OST with nodsh" && return 0
 
@@ -1100,26 +1090,39 @@ test_26a() {      # was test_26 bug 5921 - evict dead exports by pinger
 }
 run_test 26a "evict dead exports"
 
+wait_client_evicted () {
+       local facet=$1
+       local exports=$2
+       local timeout=$3
+       local varsvc=${facet}_svc
+
+       wait_update_cond $(facet_active_host $facet) \
+               "lctl get_param -n *.${!varsvc}.num_exports | cut -d' ' -f2" \
+               "-le" $((exports - 1)) $timeout
+}
+
 test_26b() {      # bug 10140 - evict dead exports by pinger
        remote_ost_nodsh && skip "remote OST with nodsh" && return 0
 
-       if [ $(facet_host mgs) = $(facet_host ost1) ]; then
+       [[ $(facet_host mgs) != $(facet_host ost1) ]] ||
                skip "msg and ost1 are at the same node"
-               return 0
-       fi
 
        check_timeout || return 1
        clients_up
-       zconf_mount `hostname` $MOUNT2 ||
+       zconf_mount $HOSTNAME $MOUNT2 ||
                 { error "Failed to mount $MOUNT2"; return 2; }
-       sleep 1 # wait connections being established
+       # make sure all imports are connected and not IDLE
+       do_facet client $LFS df > /dev/null
+       $LFS setstripe -c -1 $MOUNT/$tfile
 
-       local MDS_NEXP=$(do_facet $SINGLEMDS lctl get_param -n mdt.${mds1_svc}.num_exports | cut -d' ' -f2)
-       local OST_NEXP=$(do_facet ost1 lctl get_param -n obdfilter.${ost1_svc}.num_exports | cut -d' ' -f2)
+       local mds_nexp=$(do_facet mds1 \
+               lctl get_param -n mdt.${mds1_svc}.num_exports)
+       local ost_nexp=$(do_facet ost1 \
+               lctl get_param -n obdfilter.${ost1_svc}.num_exports)
 
-       echo starting with $OST_NEXP OST and $MDS_NEXP MDS exports
+       echo "starting with '$ost_nexp' OST and '$mds_nexp' MDS exports"
 
-       zconf_umount `hostname` $MOUNT2 -f
+       zconf_umount $HOSTNAME $MOUNT2 -f
 
        # PING_INTERVAL max(obd_timeout / 4, 1U)
        # PING_EVICT_TIMEOUT (PING_INTERVAL * 6)
@@ -1130,10 +1133,10 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
        # = 6 * PING_INTERVAL + PING_INTERVAL
        # = 7 PING_INTERVAL = 7 obd_timeout / 4 =  (1+3/4)obd_timeout
        # let's wait $((TIMEOUT * 2)) # bug 19887
-       wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 2)) ||
-               error "Client was not evicted by ost"
-       wait_client_evicted $SINGLEMDS $MDS_NEXP $((TIMEOUT * 2)) ||
-               error "Client was not evicted by mds"
+       wait_client_evicted ost1 $ost_nexp $((TIMEOUT * 2)) ||
+               error "Client was not evicted by OSS"
+       wait_client_evicted mds1 $mds_nexp $((TIMEOUT * 2)) ||
+               error "Client was not evicted by MDS"
 }
 run_test 26b "evict dead exports"