lustre/tests/recovery-mds-scale.sh

   1 #!/bin/bash
   2 #
   3 # Was Test 11 in cmd3.
   4 # For duration of 24 hours repeatedly failover a random MDS at
   5 # 10 minute intervals and verify that no application errors occur.
   6
   7 # Test runs one of CLIENT_LOAD progs on remote clients.
   8 set -e
   9
  10 ONLY=${ONLY:-"$*"}
  11
  12 LUSTRE=${LUSTRE:-$(dirname $0)/..}
  13 . $LUSTRE/tests/test-framework.sh
  14 init_test_env "$@"
  15 init_logging
  16
  17 # bug number for skipped test:
  18 ALWAYS_EXCEPT="$RECOVERY_MDS_SCALE_EXCEPT"
  19 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
  20
  21 build_test_filter
  22
  23 remote_mds_nodsh && skip_env "remote MDS with nodsh"
  24 remote_ost_nodsh && skip_env "remote OST with nodsh"
  25
  26 (( $CLIENTCOUNT < 3 )) &&
  27         skip_env "need three or more clients"
  28
  29 # SHARED_DIRECTORY should be specified with a shared directory which is
  30 # accessable on all of the nodes
  31 if [ -z "$SHARED_DIRECTORY" ] || ! check_shared_dir $SHARED_DIRECTORY; then
  32         skip_env "SHARED_DIRECTORY not set"
  33 fi
  34
  35 ERRORS_OK=""    # No application failures should occur during this test.
  36
  37 if [ "$SLOW" = "no" ]; then
  38         DURATION=${DURATION:-$((60 * 30))}
  39 else
  40         DURATION=${DURATION:-$((60 * 60 * 24))}
  41 fi
  42 SERVER_FAILOVER_PERIOD=${SERVER_FAILOVER_PERIOD:-$((60 * 10))} # 10 minutes
  43
  44 MINSLEEP=${MINSLEEP:-120}
  45 REQFAIL_PERCENT=${REQFAIL_PERCENT:-3}    # bug17839 comment 62
  46 # round up the result of integer division: C=(A + (B - 1)) / B
  47 REQFAIL=${REQFAIL:-$(((DURATION * REQFAIL_PERCENT + (SERVER_FAILOVER_PERIOD *
  48         100 - 1 )) / SERVER_FAILOVER_PERIOD / 100))}
  49
  50 END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
  51 LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
  52 VMSTAT_PID_FILE=${VMSTAT_PID_FILE:-$TMP/vmstat.pid}
  53
  54 run_info() {
  55         local oput=echo
  56
  57         $oput "******* Run Information *******"
  58         $oput "SERVER_FAILOVER_PERIOD:$1"
  59         $oput "DURATION:$2"
  60         $oput "MINSLEEP:$3"
  61         $oput "SLOW:$4"
  62         $oput "REQFAIL:$5"
  63         $oput "SHARED_DIRECTORY:$6"
  64         $oput "END_RUN_FILE:$7"
  65         $oput "LOAD_PID_FILE:$8"
  66         $oput "VMSTAT_PID_FILE:$9"
  67         $oput "CLIENTCOUNT:$10"
  68         $oput "MDTS:$11"
  69         $oput "OSTS:$12"
  70         $oput "*******************************"
  71 }
  72
  73 server_numfailovers () {
  74         local facet=$1
  75         local var=${facet}_numfailovers
  76         local val=0
  77
  78         [[ ${!var} ]] && val=${!var}
  79         echo $val
  80 }
  81
  82 servers_numfailovers () {
  83         local facet
  84         local var
  85
  86         for facet in ${MDTS//,/ } ${OSTS//,/ }; do
  87                 echo "$facet: $(server_numfailovers $facet) times"
  88         done
  89 }
  90
  91 summary_and_cleanup () {
  92         local rc=$?
  93
  94         # Having not empty END_RUN_FILE means the failed loads only
  95         if [ -s $END_RUN_FILE ]; then
  96                 print_end_run_file $END_RUN_FILE
  97                 rc=1
  98         fi
  99
 100         echo $(date +'%F %H:%M:%S') Terminating clients loads ...
 101         echo "$0" >> $END_RUN_FILE
 102         local result=PASS
 103         (( $rc == 0 )) || result=FAIL
 104
 105         log "Duration:               $DURATION"
 106         log "Server failover period: $SERVER_FAILOVER_PERIOD seconds"
 107         log "Exited after:           $ELAPSED seconds"
 108         log "Number of failovers before exit:"
 109         log "$(servers_numfailovers)"
 110         log "Status: $result: rc=$rc"
 111
 112         # stop vmstat on OSS nodes
 113         [ "$VMSTAT" ] &&
 114                 stop_process $(comma_list $(osts_nodes)) $VMSTAT_PID_FILE
 115
 116         # stop the client loads
 117         stop_client_loads $NODES_TO_USE $LOAD_PID_FILE
 118
 119         if (( $rc != 0 )); then
 120                 # we are interested in only on failed clients and servers
 121                 local failedclients=$(cat $END_RUN_FILE | grep -v $0)
 122                 gather_logs $(comma_list $(all_server_nodes) $failedclients)
 123         fi
 124
 125         exit $rc
 126 }
 127
 128 failover_target() {
 129         local flavor=${1:-"MDS"}
 130         local servers
 131         local serverfacet
 132         local var
 133
 134         [ "$flavor" = "MDS" ] && servers=$MDTS || servers=$OSTS
 135
 136         stack_trap summary_and_cleanup EXIT INT
 137
 138         # start vmstat on OSS nodes
 139         [ "$VMSTAT" ] &&
 140                 start_vmstat $(comma_list $(osts_nodes)) $VMSTAT_PID_FILE
 141
 142         # start client loads
 143         rm -f $END_RUN_FILE
 144         start_client_loads $NODES_TO_USE
 145
 146         echo client loads pids:
 147         do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE" || exit 3
 148
 149         ELAPSED=0
 150         local sleep=0
 151         local reqfail=0
 152         local it_time_start
 153         local start_ts=$(date +%s)
 154         local current_ts=$start_ts
 155
 156         while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
 157                 # In order to perform the
 158                 # expected number of failovers, we need to account the
 159                 # following:
 160                 # 1) the time that has elapsed during the client load checking
 161                 # 2) time takes for failover
 162                 it_time_start=$(date +%s)
 163
 164                 serverfacet=$(get_random_entry $servers)
 165                 var=${serverfacet}_numfailovers
 166
 167                 # Check that our client loads are still running. If any have
 168                 # died, that means they have died outside of recovery, which
 169                 # is unacceptable.
 170                 log "==== Check clients loads BEFORE failover -- failure NOT OK \
 171                      ELAPSED=$ELAPSED DURATION=$DURATION \
 172                      PERIOD=$SERVER_FAILOVER_PERIOD"
 173                 check_client_loads $NODES_TO_USE || exit 4
 174
 175                 log "Wait $serverfacet recovery complete before next failover"
 176                 if ! wait_recovery_complete $serverfacet; then
 177                     echo "$serverfacet recovery is not completed!"
 178                     exit 7
 179                 fi
 180
 181                 local want="FULL\|IDLE"
 182                 log "Checking clients are in $want state before next failover"
 183                 wait_clients_import_state $NODES_TO_USE $serverfacet "$want" ||
 184                         echo "Client import not $want, please consider to" \
 185                         "increase SERVER_FAILOVER_PERIOD=" \
 186                         "$SERVER_FAILOVER_PERIOD!"
 187
 188                 log "Starting failover on $serverfacet"
 189                 facet_failover "$serverfacet" || exit 1
 190
 191                 # Check that our client loads are still running during failover.
 192                 # No application failures should occur.
 193                 log "==== Check clients loads AFTER failover -- failure NOT OK"
 194                 if ! check_client_loads $NODES_TO_USE; then
 195                         log "Client load failed during failover. Exiting..."
 196                         exit 5
 197                 fi
 198
 199                 # Increment the number of failovers.
 200                 val=$((${!var} + 1))
 201                 eval $var=$val
 202
 203                 current_ts=$(date +%s)
 204                 ELAPSED=$((current_ts - start_ts))
 205
 206                 sleep=$((SERVER_FAILOVER_PERIOD - (current_ts - it_time_start)))
 207
 208                 # Keep counting the number of iterations when
 209                 # time spent to failover and two client loads check exceeded
 210                 # the value ( SERVER_FAILOVER_PERIOD - MINSLEEP ).
 211                 if (( $sleep < $MINSLEEP )); then
 212                         reqfail=$((reqfail + 1))
 213
 214                         cat <<- END_WARN
 215                         WARNING: failover and two check_client_loads time exceeded: SERVER_FAILOVER_PERIOD - MINSLEEP!
 216                         Failed loading I/O for a min period of $MINSLEEP $reqfail times (REQFAIL=$REQFAIL)
 217                         This iteration, load only applied for sleep=$sleep seconds.
 218                         Estimated max recovery time: $MAX_RECOV_TIME
 219                         Probably hardware is taking excessively long time to boot.
 220                         Try increasing SERVER_FAILOVER_PERIOD (current is $SERVER_FAILOVER_PERIOD), bug 20918
 221                         END_WARN
 222                         (( $reqfail > $REQFAIL )) && exit 6
 223                 fi
 224
 225                 log "$serverfacet failed over ${!var} times, and counting..."
 226
 227                 [ $((ELAPSED + sleep)) -ge $DURATION ] && break
 228
 229                 if (( $sleep > 0 )); then
 230                         echo "sleeping $sleep seconds... "
 231                         sleep $sleep
 232                 fi
 233         done
 234         exit 0
 235 }
 236
 237 ################################## Main Flow ###################################
 238 check_and_setup_lustre
 239 rm -rf $DIR/[Rdfs][0-9]*
 240
 241
 242 MAX_RECOV_TIME=$(max_recovery_time)
 243
 244 # The test node needs to be insulated from a lustre failure as much as possible,
 245 # so not even loading the lustre modules is ideal.
 246 # -- umount lustre
 247 # -- remove hostname from clients list
 248 zconf_umount $HOSTNAME $MOUNT
 249 NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
 250 NODES_TO_USE=$(exclude_items_from_list $NODES_TO_USE $HOSTNAME)
 251
 252 check_progs_installed $NODES_TO_USE "${CLIENT_LOADS[@]}"
 253
 254 MDTS=$(get_facets MDS)
 255 OSTS=$(get_facets OST)
 256
 257 # Print informaiton about settings
 258 run_info $SERVER_FAILOVER_PERIOD $DURATION $MINSLEEP $SLOW $REQFAIL \
 259         $SHARED_DIRECTORY $END_RUN_FILE $LOAD_PID_FILE $VMSTAT_PID_FILE \
 260         $CLIENTCOUNT $MDTS $OSTS
 261
 262 test_failover_mds() {
 263         # failover a random MDS
 264         failover_target MDS
 265 }
 266 run_test failover_mds "failover MDS"
 267
 268 test_failover_ost() {
 269         # failover a random OST
 270         failover_target OST
 271 }
 272 run_test failover_ost "failover OST"
 273
 274 zconf_mount $HOSTNAME $MOUNT || error "mount $MOUNT on $HOSTNAME failed"
 275 client_up || error "start client on $HOSTNAME failed"
 276
 277 complete $SECONDS
 278 check_and_cleanup_lustre
 279 exit_status