LU-6609 test: wait for import state FULL

recovery-small 26a sometimes couldn't remove sub-test dirs.
Decrement of export number may be caused by net issues.
So, now test is passed only when import state becomes EVICTED.
And in the end it waits for state FULL before removing sub-test dirs.

Test-Parameters: trivial envdefinitions=SLOW=yes,ONLY=26a testlist=recovery-small

Change-Id: Ib6156f4761bc79d89b42654898b51cc86c2ef40a
Reviewed-on: http://es-gerrit.xyus.xyratex.com:8080/1277
Signed-off-by: Sergey Cheremencev <Sergey_Cheremencev@xyratex.com>
Xyratex-bug-id: MRP-1168
Tested-by: Jenkins
Tested-by: Elena Gryaznova <elena_gryaznova@xyratex.com>
Reviewed-by: Alexey Lyashkov <alexey_lyashkov@xyratex.com>
Reviewed-by: Andrew Perepechko <andrew_perepechko@xyratex.com>
Reviewed-by: Vitaly Fertman <vitaly_fertman@xyratex.com>
Reviewed-on: https://review.whamcloud.com/14843
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 271aae5..c311e81 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -1076,20 +1076,21 @@ test_26a() {      # was test_26 bug 5921 - evict dead exports by pinger
 
        check_timeout || return 1
 
-       local OST_NEXP=$(do_facet ost1 lctl get_param -n obdfilter.${ost1_svc}.num_exports | cut -d' ' -f2)
-
-       echo starting with $OST_NEXP OST exports
 # OBD_FAIL_PTLRPC_DROP_RPC 0x505
        do_facet client lctl set_param fail_loc=0x505
-        # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
-        # But if there's a race to start the evictor from various obds,
-        # the loser might have to wait for the next ping.
-
+       local before=$(date +%s)
        local rc=0
-       wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 2 + TIMEOUT * 3 / 4))
-       rc=$?
+
+       # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
+       # But if there's a race to start the evictor from various obds,
+       # the loser might have to wait for the next ping.
+       sleep $((TIMEOUT * 2 + TIMEOUT * 3 / 4))
        do_facet client lctl set_param fail_loc=0x0
-        [ $rc -eq 0 ] || error "client not evicted from OST"
+       do_facet client df > /dev/null
+
+       local oscs=$(lctl dl | awk '/-osc-/ {print $4}')
+       check_clients_evicted $before ${oscs[@]}
+       check_clients_full 10 ${oscs[@]}
 }
 run_test 26a "evict dead exports"
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 4c29a97..509041d 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -8448,3 +8448,41 @@ lfsck_verify_pfid()
               "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=0"
        return $rc
 }
+
+# check that clients "oscs" was evicted after "before"
+check_clients_evicted() {
+       local before=$1
+       shift
+       local oscs=${@}
+       local osc
+       local rc=0
+
+       for osc in $oscs; do
+               ((rc++))
+               echo "Check state for $osc"
+               local evicted=$(do_facet client $LCTL get_param osc.$osc.state |
+                       tail -n 3 | awk -F"[ [,]" \
+                       '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+               if (($? == 0)) && (($evicted > $before)); then
+                       echo "$osc is evicted at $evicted"
+                       ((rc--))
+               fi
+       done
+
+       [ $rc -eq 0 ] || error "client not evicted from OST"
+}
+
+# check that clients OSCS current_state is FULL
+check_clients_full() {
+       local timeout=$1
+       shift
+       local oscs=${@}
+
+       for osc in $oscs; do
+               wait_update_facet client \
+                       "lctl get_param -n osc.$osc.state |
+                       grep 'current_state: FULL'" \
+                       "current_state: FULL" $timeout
+               [ $? -eq 0 ] || error "$osc state is not FULL"
+       done
+}