2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
13 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
15 if [ "$FAILURE_MODE" = "HARD" ]; then
16 mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \
17 echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \
18 echo "Except the tests: $CONFIG_EXCEPTIONS" && \
19 ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
23 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
26 CLEANUP=${CLEANUP:-""}
30 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
31 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
32 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
34 assert_env mds_HOST MDS_MKFS_OPTS
35 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
36 assert_env LIVE_CLIENT FSNAME
38 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
39 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
41 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
42 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
47 # fail clients round robin
49 # list of failable clients
50 FAIL_LIST=($FAIL_CLIENTS)
51 FAIL_NUM=${#FAIL_LIST[*]}
54 DOWN_NUM=0 # number of nodes currently down
56 # set next client to fail
58 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
59 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
60 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
65 if [ "$FAILURE_MODE" = HARD ]; then
67 while ping -w 3 -c 1 $client > /dev/null 2>&1; do
68 echo "waiting for node $client to fail"
71 elif [ "$FAILURE_MODE" = SOFT ]; then
72 zconf_umount $client $MOUNT -f
79 log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
80 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
81 num=$((FAIL_NUM - DOWN_NUM))
84 if [ -z "$num" ] || [ "$num" -le 0 ]; then
85 log "No clients failed!"
91 for i in `seq $num`; do
94 DOWN_CLIENTS="$DOWN_CLIENTS $client"
95 shutdown_client $client
98 echo "down clients: $DOWN_CLIENTS"
100 for client in $DOWN_CLIENTS; do
103 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
107 reintegrate_clients() {
108 for client in $DOWN_CLIENTS; do
109 wait_for_host $client
110 echo "Restarting $client"
111 zconf_mount $client $MOUNT || return 1
118 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
125 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
126 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
127 $PDSH $c touch $MOUNT/${c}_$file || return 1
133 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
134 $PDSH $c rm $MOUNT/${c}_$file
139 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
140 echo "$c mkdir $MOUNT/$c"
141 $PDSH $c "mkdir $MOUNT/$c"
142 $PDSH $c "ls -l $MOUNT/$c"
147 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
148 echo "rmdir $MOUNT/$c"
149 $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c"
153 clients_recover_osts() {
155 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
158 check_and_setup_lustre
160 # 9 Different Failure Modes Combinations
161 echo "Starting Test 17 at `date`"
164 facet_failover $SINGLEMDS
165 echo "Waiting for df pid: $DFPID"
166 wait $DFPID || { echo "df returned $?" && return 1; }
168 for i in $(seq $OSTCOUNT) ; do
169 facet_failover ost$i || return 4
170 echo "Waiting for df pid: $DFPID"
171 wait $DFPID || { echo "df returned $?" && return 3; }
175 run_test 0 "Fail all nodes, independently"
177 ############### First Failure Mode ###############
179 echo "Don't do a MDS - MDS Failure Case"
180 echo "This makes no sense"
182 run_test 1 "MDS/MDS failure"
183 ###################################################
185 ############### Second Failure Mode ###############
187 echo "Verify Lustre filesystem is up and running"
188 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
192 shutdown_facet $SINGLEMDS
193 reboot_facet $SINGLEMDS
195 # prepare for MDS failover
196 change_active $SINGLEMDS
197 reboot_facet $SINGLEMDS
205 echo "Reintegrating OST"
208 start_ost 1 || return 2
211 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
215 clients_recover_osts ost1
216 echo "Verify reintegration"
217 client_df || return 1
220 run_test 2 "Second Failure Mode: MDS/OST `date`"
221 ###################################################
224 ############### Third Failure Mode ###############
227 echo "Verify Lustre filesystem is up and running"
228 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
231 facet_failover $SINGLEMDS
232 wait $DFPID || echo df failed: $?
235 echo "Test Lustre stability after MDS failover"
239 echo "Failing 2 CLIENTS"
243 echo "Test Lustre stability after CLIENT failure"
247 echo "Reintegrating CLIENTS"
248 reintegrate_clients || return 1
250 client_df || return 3
251 sleep 2 # give it a little time for fully recovered before next test
253 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
254 ###################################################
256 ############### Fourth Failure Mode ###############
258 echo "Fourth Failure Mode: OST/MDS `date`"
264 echo "Test Lustre stability after OST failure"
270 shutdown_facet $SINGLEMDS
271 reboot_facet $SINGLEMDS
273 # prepare for MDS failover
274 change_active $SINGLEMDS
275 reboot_facet $SINGLEMDS
282 echo "Reintegrating OST"
288 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
293 clients_recover_osts ost1
294 echo "Test Lustre stability after MDS failover"
295 client_df || return 1
297 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
298 ###################################################
300 ############### Fifth Failure Mode ###############
302 [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0
304 echo "Fifth Failure Mode: OST/OST `date`"
307 echo "Verify Lustre filesystem is up and running"
308 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
317 echo "Test Lustre stability after OST failure"
327 echo "Test Lustre stability after OST failure"
333 echo "Reintegrating OSTs"
339 clients_recover_osts ost1
340 clients_recover_osts ost2
345 client_df || return 2
347 run_test 5 "Fifth Failure Mode: OST/OST `date`"
348 ###################################################
350 ############### Sixth Failure Mode ###############
352 echo "Sixth Failure Mode: OST/CLIENT `date`"
355 echo "Verify Lustre filesystem is up and running"
356 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
359 client_touch testfile || return 2
366 echo "Test Lustre stability after OST failure"
373 echo "Failing CLIENTs"
377 echo "Test Lustre stability after CLIENTs failure"
384 echo "Reintegrating OST/CLIENTs"
387 reintegrate_clients || return 1
390 wait_remote_prog df $((TIMEOUT * 3 + 10))
394 echo "Verifying mount"
395 [ -z "$(mounted_lustre_filesystems)" ] && return 3
398 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
399 ###################################################
402 ############### Seventh Failure Mode ###############
404 echo "Seventh Failure Mode: CLIENT/MDS `date`"
407 echo "Verify Lustre filesystem is up and running"
408 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
411 client_touch testfile || return 1
414 echo "Part 1: Failing CLIENT"
418 echo "Test Lustre stability after CLIENTs failure"
420 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
421 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
424 echo "Wait 1 minutes"
428 echo "Verify Lustre filesystem is up and running"
429 [ -z "$(mounted_lustre_filesystems)" ] && return 2
435 facet_failover $SINGLEMDS
438 echo "Test Lustre stability after MDS failover"
439 wait $DFPID || echo "df on down clients fails " || return 1
440 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
441 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
444 echo "Reintegrating CLIENTs"
445 reintegrate_clients || return 2
449 echo "wait 1 minutes"
452 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
453 ###################################################
456 ############### Eighth Failure Mode ###############
458 echo "Eighth Failure Mode: CLIENT/OST `date`"
461 echo "Verify Lustre filesystem is up and running"
462 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
465 client_touch testfile
468 echo "Failing CLIENTs"
472 echo "Test Lustre stability after CLIENTs failure"
474 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
475 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
478 echo "Wait 1 minutes"
482 echo "Verify Lustre filesystem is up and running"
483 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
486 client_touch testfile
494 echo "Test Lustre stability after OST failure"
498 #non-failout hangs forever here
499 #$PDSH $LIVE_CLIENT "ls -l $MOUNT"
500 #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
503 echo "Reintegrating CLIENTs/OST"
504 reintegrate_clients || return 3
508 client_df || return 1
509 client_touch testfile2 || return 2
512 echo "Wait 1 minutes"
515 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
516 ###################################################
519 ############### Ninth Failure Mode ###############
524 echo "Verify Lustre filesystem is up and running"
525 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
528 client_touch testfile || return 1
531 echo "Failing CLIENTs"
535 echo "Test Lustre stability after CLIENTs failure"
537 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
538 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
541 echo "Wait 1 minutes"
545 echo "Verify Lustre filesystem is up and running"
546 $PDSH $LIVE_CLIENT df $MOUNT || return 3
547 client_touch testfile || return 4
550 echo "Failing CLIENTs"
554 echo "Test Lustre stability after CLIENTs failure"
556 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
557 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
560 echo "Reintegrating CLIENTs/CLIENTs"
561 reintegrate_clients || return 7
565 echo "Wait 1 minutes"
568 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
569 ###################################################
572 #Run availability after all failures
573 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
574 LOADTEST=${LOADTEST:-metadata-load.py}
575 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
577 run_test 10 "Running Availability for 6 hours..."
579 equals_msg `basename $0`: test complete, cleaning up
580 check_and_cleanup_lustre
581 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true