X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Frecovery-mds-scale.sh;h=675988c8da5693a44cee808153088d5c972a62e7;hp=fd8d32fd4326bae44a76394dcc1d549ac99e564f;hb=3b524b627b87ac0f1355658dbbe2e4a16cda122b;hpb=3131ab26bcfe548b6f8a5ba709460af9c88ae044

diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh
index fd8d32f..675988c 100644
--- a/lustre/tests/recovery-mds-scale.sh
+++ b/lustre/tests/recovery-mds-scale.sh
@@ -15,8 +15,11 @@ init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-TESTSUITELOG=${TESTSUITELOG:-$TMP/recovery-mds-scale}
+TESTSUITELOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh)}
 DEBUGLOG=$TESTSUITELOG.debug
+
+cleanup_logs
+
 exec 2>$DEBUGLOG
 echo "--- env ---" >&2
 env >&2
@@ -28,7 +31,7 @@ set -x
 
 [ -n "$CLIENTS" ] || { skip "$0 Need two or more remote clients" && exit 0; }
 [ $CLIENTCOUNT -ge 3 ] || \
-    { skip "$0 Need two or more clients, have $CLIENTCOUNT" && exit 0; }
+    { skip "$0 Need two or more remote clients, have $CLIENTCOUNT" && exit 0; }
 
 END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
 LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
@@ -145,6 +148,15 @@ Status: $result: rc=$rc"
         sleep 5
         kill -9 $CLIENT_LOAD_PIDS || true
     fi
+    if [ $rc -ne 0 ]; then
+        # we are interested in only on failed clients and servers
+        local failedclients=$(cat $END_RUN_FILE | grep -v $0)
+        # FIXME: need ostfailover-s nodes also for FLAVOR=OST
+        local product=$(gather_logs $(comma_list $(osts_nodes) \
+                               $(mdts_nodes) $mdsfailover_HOST $failedclients))
+        echo logs files $product
+    fi
+
     [ $rc -eq 0 ] && zconf_mount $(hostname) $MOUNT
 
     exit $rc
@@ -207,6 +219,19 @@ while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
         exit 4
     fi
 
+    log "Wait $SERVERFACET recovery complete before doing next failover ...."
+    if [[ $NUM_FAILOVERS != 0 ]]; then
+        if ! wait_recovery_complete $SERVERFACET ; then
+            echo "$SERVERFACET recovery is not completed!"
+            exit 7
+        fi
+    fi
+
+    log "Checking clients are in FULL state before doing next failover"
+    if ! wait_clients_import_state $NODES_TO_USE $SERVERFACET FULL; then
+        echo "Clients import not FULL, please consider to increase SERVER_FAILOVER_PERIOD=$SERVER_FAILOVER_PERIOD !"
+        
+    fi
     log "Starting failover on $SERVERFACET"
 
     facet_failover "$SERVERFACET" || exit 1