2 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 # For duration of 24 hours repeatedly failover a random MDS at
6 # 10 minute intervals and verify that no application errors occur.
8 # Test runs one of CLIENT_LOAD progs on remote clients.
13 # bug number for skipped test:
14 ALWAYS_EXCEPT="$RECOVERY_MDS_SCALE_EXCEPT"
15 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
17 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
18 . $LUSTRE/tests/test-framework.sh
20 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
23 remote_mds_nodsh && skip_env "remote MDS with nodsh" && exit 0
24 remote_ost_nodsh && skip_env "remote OST with nodsh" && exit 0
26 [ -z "$CLIENTS" -o $CLIENTCOUNT -lt 3 ] &&
27 skip_env "need three or more clients" && exit 0
29 if [ -z "$SHARED_DIRECTORY" ] || ! check_shared_dir $SHARED_DIRECTORY; then
30 skip_env "SHARED_DIRECTORY should be specified with a shared directory \
31 which is accessable on all of the nodes"
35 ERRORS_OK="" # No application failures should occur during this test.
37 if [ "$SLOW" = "no" ]; then
38 DURATION=${DURATION:-$((60 * 30))}
40 DURATION=${DURATION:-$((60 * 60 * 24))}
42 SERVER_FAILOVER_PERIOD=${SERVER_FAILOVER_PERIOD:-$((60 * 10))} # 10 minutes
44 MINSLEEP=${MINSLEEP:-120}
45 REQFAIL_PERCENT=${REQFAIL_PERCENT:-3} # bug17839 comment 62
46 REQFAIL=${REQFAIL:-$((DURATION / SERVER_FAILOVER_PERIOD *
47 REQFAIL_PERCENT / 100))}
49 END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
50 LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
51 VMSTAT_PID_FILE=${VMSTAT_PID_FILE:-$TMP/vmstat.pid}
53 server_numfailovers () {
55 local var=${facet}_numfailovers
58 [[ ${!var} ]] && val=${!var}
62 servers_numfailovers () {
66 for facet in ${MDTS//,/ } ${OSTS//,/ }; do
67 echo "$facet: $(server_numfailovers $facet) times"
71 summary_and_cleanup () {
75 # Having not empty END_RUN_FILE means the failed loads only
76 if [ -s $END_RUN_FILE ]; then
77 print_end_run_file $END_RUN_FILE
81 echo $(date +'%F %H:%M:%S') Terminating clients loads ...
82 echo "$0" >> $END_RUN_FILE
84 [ $rc -eq 0 ] || result=FAIL
86 log "Duration: $DURATION
87 Server failover period: $SERVER_FAILOVER_PERIOD seconds
88 Exited after: $ELAPSED seconds
89 Number of failovers before exit:
90 $(servers_numfailovers)
91 Status: $result: rc=$rc"
93 # stop vmstat on OSS nodes
94 [ "$VMSTAT" ] && stop_process $(comma_list $(osts_nodes)) $VMSTAT_PID_FILE
96 # stop the client loads
97 stop_client_loads $NODES_TO_USE $LOAD_PID_FILE
99 if [ $rc -ne 0 ]; then
100 # we are interested in only on failed clients and servers
101 local failedclients=$(cat $END_RUN_FILE | grep -v $0)
102 # FIXME: need ostfailover-s nodes also for FLAVOR=OST
103 gather_logs $(comma_list $(osts_nodes) $(mdts_nodes) \
104 $mdsfailover_HOST $failedclients)
111 local flavor=${1:-"MDS"}
116 [ "$flavor" = "MDS" ] && servers=$MDTS || servers=$OSTS
118 trap summary_and_cleanup EXIT INT
120 # start vmstat on OSS nodes
121 [ "$VMSTAT" ] && start_vmstat $(comma_list $(osts_nodes)) $VMSTAT_PID_FILE
125 start_client_loads $NODES_TO_USE
127 echo client loads pids:
128 do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE" || exit 3
134 local start_ts=$(date +%s)
135 local current_ts=$start_ts
137 while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
138 # In order to perform the
139 # expected number of failovers, we need to account the following:
140 # 1) the time that has elapsed during the client load checking
141 # 2) time takes for failover
142 it_time_start=$(date +%s)
144 serverfacet=$(get_random_entry $servers)
145 var=${serverfacet}_numfailovers
147 # Check that our client loads are still running. If any have died,
148 # that means they have died outside of recovery, which is unacceptable.
149 log "==== Checking the clients loads BEFORE failover -- failure NOT OK \
150 ELAPSED=$ELAPSED DURATION=$DURATION PERIOD=$SERVER_FAILOVER_PERIOD"
151 check_client_loads $NODES_TO_USE || exit 4
153 log "Wait $serverfacet recovery complete before doing next failover..."
154 if ! wait_recovery_complete $serverfacet; then
155 echo "$serverfacet recovery is not completed!"
159 log "Checking clients are in FULL state before doing next failover..."
160 if ! wait_clients_import_state $NODES_TO_USE $serverfacet FULL; then
161 echo "Clients import not FULL, please consider to increase \
162 SERVER_FAILOVER_PERIOD=$SERVER_FAILOVER_PERIOD!"
165 log "Starting failover on $serverfacet"
166 facet_failover "$serverfacet" || exit 1
168 # Check that our client loads are still running during failover.
169 # No application failures should occur.
170 log "==== Checking the clients loads AFTER failover -- failure NOT OK"
171 if ! check_client_loads $NODES_TO_USE; then
172 log "Client load failed during failover. Exiting..."
176 # Increment the number of failovers.
180 current_ts=$(date +%s)
181 ELAPSED=$((current_ts - start_ts))
183 sleep=$((SERVER_FAILOVER_PERIOD - (current_ts - it_time_start)))
185 # Keep counting the number of iterations when
186 # time spent to failover and two client loads check exceeded
187 # the value ( SERVER_FAILOVER_PERIOD - MINSLEEP ).
188 if [ $sleep -lt $MINSLEEP ]; then
189 reqfail=$((reqfail + 1))
190 log "WARNING: failover and two check_client_loads time exceeded \
191 SERVER_FAILOVER_PERIOD - MINSLEEP!
192 Failed to load the filesystem with I/O for a minimum period of \
193 $MINSLEEP $reqfail times ( REQFAIL=$REQFAIL ).
194 This iteration, the load was only applied for sleep=$sleep seconds.
195 Estimated max recovery time: $MAX_RECOV_TIME
196 Probably the hardware is taking excessively long time to boot.
197 Try to increase SERVER_FAILOVER_PERIOD (current is $SERVER_FAILOVER_PERIOD), \
199 [ $reqfail -gt $REQFAIL ] && exit 6
202 log "$serverfacet has failed over ${!var} times, and counting..."
204 [ $((ELAPSED + sleep)) -ge $DURATION ] && break
206 if [ $sleep -gt 0 ]; then
207 echo "sleeping $sleep seconds... "
214 ################################## Main Flow ###################################
217 check_and_setup_lustre
218 rm -rf $DIR/[Rdfs][0-9]*
220 MAX_RECOV_TIME=$(max_recovery_time)
222 # The test node needs to be insulated from a lustre failure as much as possible,
223 # so not even loading the lustre modules is ideal.
225 # -- remove hostname from clients list
226 zconf_umount $HOSTNAME $MOUNT
227 NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
228 NODES_TO_USE=$(exclude_items_from_list $NODES_TO_USE $HOSTNAME)
230 check_progs_installed $NODES_TO_USE ${CLIENT_LOADS[@]}
232 MDTS=$(get_facets MDS)
233 OSTS=$(get_facets OST)
235 test_failover_mds() {
236 # failover a random MDS
239 run_test failover_mds "failover MDS"
241 test_failover_ost() {
242 # failover a random OST
245 run_test failover_ost "failover OST"
247 zconf_mount $HOSTNAME $MOUNT || error "mount $MOUNT on $HOSTNAME failed"
248 client_up || error "start client on $HOSTNAME failed"
251 check_and_cleanup_lustre