b=24410 exit with error if NFSCLIENT is set, but no nfs export found

[fs/lustre-release.git] / lustre / tests / recovery-mds-scale.sh

diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh
index 9b1bed2..82a5507 100644 (file)
--- a/lustre/tests/recovery-mds-scale.sh
+++ b/lustre/tests/recovery-mds-scale.sh
@@ -27,11 +27,13 @@ echo "--- env ---" >&2
 set -x
 
 [ "$SHARED_DIRECTORY" ] || \
-    { skip "$0: Empty SHARED_DIRECTORY" && exit 0; }
+    { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; }
+
+[ -n "$CLIENTS" ] || \
+    { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; }
 
-[ -n "$CLIENTS" ] || { skip "$0 Need two or more remote clients" && exit 0; }
 [ $CLIENTCOUNT -ge 3 ] || \
-    { skip "$0 Need two or more remote clients, have $CLIENTCOUNT" && exit 0; }
+    { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients, have $((CLIENTCOUNT - 1))" && exit 0; }
 
 END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
 LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
@@ -44,6 +46,8 @@ build_test_filter
 check_and_setup_lustre
 rm -rf $DIR/[df][0-9]*
 
+max_recov_time=$(max_recovery_time)
+
 # the test node needs to be insulated from a lustre failure as much as possible,
 # so not even loading the lustre modules is ideal.
 # -- umount lustre
@@ -79,15 +83,20 @@ rm -f $END_RUN_FILE
 vmstatLOG=${TESTSUITELOG}_$(basename $0 .sh).vmstat
 
 server_numfailovers () {
+    local facet=$1
+    local var=${facet}_numfailovers
+    local val=0
+
+    [[ ${!var} ]] && val=${!var}
+    echo $val
+}
+
+servers_numfailovers () {
     local facet
     local var
 
     for facet in $MDTS ${OSTS//,/ }; do
-        var=${facet}_nums
-        val=${!var}
-        if [ "$val" ] ; then
-            echo "$facet failed  over  $val times"
-        fi
+        echo "$facet: $(server_numfailovers $facet) times"
     done
 }
 
@@ -109,7 +118,7 @@ summary_and_cleanup () {
     # actually failed though.  the first node in the END_RUN_NODE is
     # the one we are really interested in.
         if [ -n "$END_RUN_NODE" ]; then
-            var=$(client_var_name $END_RUN_NODE)_load
+            var=$(node_var_name $END_RUN_NODE)_load
             echo "Client load failed on node $END_RUN_NODE" 
             echo
             echo "client $END_RUN_NODE load stdout and debug files :
@@ -124,11 +133,11 @@ summary_and_cleanup () {
     local result=PASS
     [ $rc -eq 0 ] || result=FAIL
 
-    log "Duraion:                $DURATION
+    log "Duration:                $DURATION
 Server failover period: $SERVER_FAILOVER_PERIOD seconds
 Exited after:           $ELAPSED seconds
 Number of failovers before exit:
-$(server_numfailovers)
+$(servers_numfailovers)
 Status: $result: rc=$rc"
 
     # stop the vmstats on the OSTs
@@ -170,7 +179,6 @@ log "-----============= $0 starting =============-----"
 trap summary_and_cleanup EXIT INT
 
 ELAPSED=0
-NUM_FAILOVERS=0
 
 # vmstat the osts
 if [ "$VMSTAT" ]; then
@@ -181,22 +189,19 @@ fi
 start_client_loads $NODES_TO_USE
 
 echo clients load pids:
-if ! do_nodes $NODES_TO_USE "set -x; echo \$(hostname): && cat $LOAD_PID_FILE"; then
-    if [ -e $DEBUGLOG ]; then
-        exec 2<&-
-        cat $DEBUGLOG
+if ! do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE"; then
         exit 3
-    fi
 fi
 
-START_TS=$(date +%s)
-CURRENT_TS=$START_TS
-
 MINSLEEP=${MINSLEEP:-120}
 REQFAIL_PERCENT=${REQFAIL_PERCENT:-3}  # bug17839 comment 62
 REQFAIL=${REQFAIL:-$(( DURATION / SERVER_FAILOVER_PERIOD * REQFAIL_PERCENT / 100))}
 reqfail=0
 sleep=0
+
+START_TS=$(date +%s)
+CURRENT_TS=$START_TS
+
 while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
 
     # In order to perform the 
@@ -207,7 +212,7 @@ while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
     it_time_start=$(date +%s)
     
     SERVERFACET=$(get_random_entry $SERVERS)
-    var=${SERVERFACET}_nums
+    var=${SERVERFACET}_numfailovers
 
     # Check that our client loads are still running. If any have died, 
     # that means they have died outside of recovery, which is unacceptable.    
@@ -220,11 +225,10 @@ while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
     fi
 
     log "Wait $SERVERFACET recovery complete before doing next failover ...."
-    if [[ $NUM_FAILOVERS != 0 ]]; then
-        if ! wait_recovery_complete $SERVERFACET ; then
-            echo "$SERVERFACET recovery is not completed!"
-            exit 7
-        fi
+
+    if ! wait_recovery_complete $SERVERFACET ; then
+        echo "$SERVERFACET recovery is not completed!"
+        exit 7
     fi
 
     log "Checking clients are in FULL state before doing next failover"
@@ -246,7 +250,6 @@ while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
     fi
 
     # Increment the number of failovers
-    NUM_FAILOVERS=$((NUM_FAILOVERS+1))
     val=$((${!var} + 1))
     eval $var=$val
  
@@ -261,13 +264,17 @@ while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
     if [ $sleep -lt $MINSLEEP ]; then
         reqfail=$((reqfail +1))
         log "WARNING: failover and two check_client_loads time exceeded SERVER_FAILOVER_PERIOD - MINSLEEP !
-Failed to meet interval $reqfail times ( REQFAIL=$REQFAIL ); have sleep=$sleep"
+Failed to load the filesystem with I/O for a minimum period of $MINSLEEP $reqfail times ( REQFAIL=$REQFAIL ).
+This iteration, the load was only applied for sleep=$sleep seconds.
+Estimated max recovery time : $max_recov_time
+Probably the hardware is taking excessively long to boot.
+Try to increase SERVER_FAILOVER_PERIOD (current is $SERVER_FAILOVER_PERIOD), bug 20918"
         [ $reqfail -gt $REQFAIL ] && exit 6 
     fi  
 
     log "$SERVERFACET has failed over ${!var} times, and counting..."
 
-    if [ $((ELAPSED + sleep)) -gt $DURATION ]; then
+    if [ $((ELAPSED + sleep)) -ge $DURATION ]; then
          break
     fi