2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 # Test multiple failures, AKA Test 17
9 LUSTRE=${LUSTRE:-`dirname $0`/..}
10 . $LUSTRE/tests/test-framework.sh
14 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
17 ALWAYS_EXCEPT="15 $INSANITY_EXCEPT"
19 if [ "$FAILURE_MODE" = "HARD" ]; then
20 skip_env "$TESTSUITE: is not functional with FAILURE_MODE = HARD, " \
21 "please use recovery-double-scale, bz20407"
24 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
27 CLEANUP=${CLEANUP:-""}
31 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
32 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
33 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
35 assert_env mds_HOST MDSCOUNT
36 assert_env ost_HOST OSTCOUNT
37 assert_env LIVE_CLIENT FSNAME
39 require_dsh_mds || exit 0
40 require_dsh_ost || exit 0
42 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
43 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
46 TESTDIR=$DIR/d0.$TESTSUITE
49 # fail clients round robin
51 # list of failable clients
52 FAIL_LIST=($FAIL_CLIENTS)
53 FAIL_NUM=${#FAIL_LIST[*]}
56 DOWN_NUM=0 # number of nodes currently down
58 # set next client to fail
60 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
61 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
62 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
68 log "Request fail clients: $num, to fail: $FAIL_NUM, failed: $DOWN_NUM"
69 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
70 num=$((FAIL_NUM - DOWN_NUM))
73 if [ -z "$num" ] || [ "$num" -le 0 ]; then
74 log "No clients failed!"
80 for i in `seq $num`; do
83 DOWN_CLIENTS="$DOWN_CLIENTS $client"
84 shutdown_client $client
87 echo "down clients: $DOWN_CLIENTS"
89 for client in $DOWN_CLIENTS; do
92 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
96 reintegrate_clients() {
97 for client in $DOWN_CLIENTS; do
99 echo "Restarting $client"
100 zconf_mount $client $MOUNT || return 1
108 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
112 start mds$1 $(mdsdevname $1) $MDS_MOUNT_OPTS
119 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
120 echo $DOWN_CLIENTS | grep -q $c && continue
121 $PDSH $c touch $TESTDIR/${c}_$file || return 1
127 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
128 $PDSH $c rm $TESTDIR/${c}_$file
133 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
134 echo "$c mkdir $TESTDIR/$c"
135 $PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c"
140 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
141 echo "rmdir $TESTDIR/$c"
142 $PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c"
146 clients_recover_osts() {
148 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
151 check_and_setup_lustre
156 # 9 Different Failure Modes Combinations
157 echo "Starting Test 17 at `date`"
160 for i in $(seq $MDSCOUNT) ; do
164 for i in $(seq $OSTCOUNT) ; do
169 run_test 0 "Fail all nodes, independently"
171 ############### First Failure Mode ###############
173 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
180 # prepare for MDS failover
190 echo "Reintegrating MDS2"
193 start_mdt 2 || return 2
196 start_mdt 1 || return $?
200 echo "Verify reintegration"
201 clients_up || return 1
203 run_test 1 "MDS/MDS failure"
204 ###################################################
206 ############### Second Failure Mode ###############
208 echo "Verify Lustre filesystem is up and running"
209 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
213 for i in $(seq $MDSCOUNT) ; do
217 # prepare for MDS failover
228 echo "Reintegrating OST"
231 start_ost 1 || return 2
233 for i in $(seq $MDSCOUNT) ; do
235 start_mdt $i || return $?
240 clients_recover_osts ost1
241 echo "Verify reintegration"
242 clients_up || return 1
245 run_test 2 "Second Failure Mode: MDS/OST `date`"
246 ###################################################
248 ############### Third Failure Mode ###############
251 echo "Verify Lustre filesystem is up and running"
252 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
255 for i in $(seq $MDSCOUNT) ; do
260 echo "Test Lustre stability after MDS failover"
264 echo "Failing 2 CLIENTS"
268 echo "Test Lustre stability after CLIENT failure"
272 echo "Reintegrating CLIENTS"
273 reintegrate_clients || return 1
275 clients_up || return 3
276 sleep 2 # give it a little time for fully recovered before next test
278 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
279 ###################################################
281 ############### Fourth Failure Mode ###############
283 echo "Fourth Failure Mode: OST/MDS `date`"
289 echo "Test Lustre stability after OST failure"
294 for i in $(seq $MDSCOUNT) ; do
298 # prepare for MDS failover
308 echo "Reintegrating OST"
313 for i in $(seq $MDSCOUNT) ; do
315 start_mdt $i || return $?
321 clients_recover_osts ost1
322 echo "Test Lustre stability after MDS failover"
323 clients_up || return 1
325 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
326 ###################################################
328 ############### Fifth Failure Mode ###############
330 [ $OSTCOUNT -lt 2 ] && skip_env "needs >= 2 OSTs"
332 echo "Fifth Failure Mode: OST/OST `date`"
335 echo "Verify Lustre filesystem is up and running"
336 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
344 echo "Test Lustre stability after OST failure"
354 echo "Test Lustre stability after OST failure"
360 echo "Reintegrating OSTs"
366 clients_recover_osts ost1
367 clients_recover_osts ost2
372 clients_up || return 2
374 run_test 5 "Fifth Failure Mode: OST/OST `date`"
375 ###################################################
377 ############### Sixth Failure Mode ###############
379 echo "Sixth Failure Mode: OST/CLIENT `date`"
382 echo "Verify Lustre filesystem is up and running"
383 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
386 client_touch testfile || return 2
393 echo "Test Lustre stability after OST failure"
400 echo "Failing CLIENTs"
404 echo "Test Lustre stability after CLIENTs failure"
411 echo "Reintegrating OST/CLIENTs"
414 reintegrate_clients || return 1
417 wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20))
421 echo "Verifying mount"
422 [ -z "$(mounted_lustre_filesystems)" ] && return 3
425 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
426 ###################################################
429 ############### Seventh Failure Mode ###############
431 echo "Seventh Failure Mode: CLIENT/MDS `date`"
434 echo "Verify Lustre filesystem is up and running"
435 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
438 client_touch testfile || return 1
441 echo "Part 1: Failing CLIENT"
445 echo "Test Lustre stability after CLIENTs failure"
447 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
448 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
451 echo "Wait 1 minutes"
455 echo "Verify Lustre filesystem is up and running"
456 [ -z "$(mounted_lustre_filesystems)" ] && return 2
462 for i in $(seq $MDSCOUNT) ; do
466 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
467 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
470 echo "Reintegrating CLIENTs"
471 reintegrate_clients || return 2
475 echo "wait 1 minutes"
478 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
479 ###################################################
482 ############### Eighth Failure Mode ###############
484 echo "Eighth Failure Mode: CLIENT/OST `date`"
487 echo "Verify Lustre filesystem is up and running"
488 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
491 client_touch testfile
494 echo "Failing CLIENTs"
498 echo "Test Lustre stability after CLIENTs failure"
500 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
501 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
504 echo "Wait 1 minutes"
508 echo "Verify Lustre filesystem is up and running"
509 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
512 client_touch testfile
520 echo "Test Lustre stability after OST failure"
524 #non-failout hangs forever here
525 #$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
526 #$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
529 echo "Reintegrating CLIENTs/OST"
530 reintegrate_clients || return 3
534 clients_up || return 1
535 client_touch testfile2 || return 2
538 echo "Wait 1 minutes"
541 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
542 ###################################################
545 ############### Ninth Failure Mode ###############
550 echo "Verify Lustre filesystem is up and running"
551 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
554 client_touch testfile || return 1
557 echo "Failing CLIENTs"
561 echo "Test Lustre stability after CLIENTs failure"
563 $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1
564 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2
567 echo "Wait 1 minutes"
571 echo "Verify Lustre filesystem is up and running"
572 client_up $LIVE_CLIENT || return 3
573 client_touch testfile || return 4
576 echo "Failing CLIENTs"
580 echo "Test Lustre stability after CLIENTs failure"
582 $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5
583 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6
586 echo "Reintegrating CLIENTs/CLIENTs"
587 reintegrate_clients || return 7
591 echo "Wait 1 minutes"
594 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
595 ###################################################
597 ############### Tenth Failure Mode ###############
599 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
604 # prepare for MDS failover
614 echo "Reintegrating OST"
617 start_ost 1 || return 2
622 # prepare for MDS failover
627 start_mdt 1 || return $?
630 start_mdt 2 || return $?
634 clients_recover_osts ost1
635 echo "Verify reintegration"
636 clients_up || return 1
638 run_test 10 "Tenth Failure Mode: MDT0/OST/MDT1 `date`"
639 ###################################################
641 ############### Seventh Failure Mode ###############
643 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
644 echo "Verify Lustre filesystem is up and running"
645 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
651 echo "Test Lustre stability after MDS failover"
655 echo "Failing 2 CLIENTS"
659 echo "Test Lustre stability after CLIENT failure"
663 echo "Reintegrating CLIENTS"
664 reintegrate_clients || return 1
668 clients_up || return 3
669 sleep 2 # give it a little time for fully recovered before next test
671 run_test 11 "Eleventh Failure Mode: MDS0/CLIENT/MDS1 `date`"
672 ###################################################
675 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
676 echo "Verify Lustre filesystem is up and running"
677 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
688 echo "Failing 2 CLIENTS"
692 echo "Test Lustre stability after CLIENT failure"
696 echo "Reintegrating CLIENTS"
697 reintegrate_clients || return 1
699 clients_up || return 3
700 sleep 2 # give it a little time for fully recovered before next test
702 run_test 12 "Twelve Failure Mode: MDS0,MDS1/OST0, OST1/CLIENTS `date`"
703 ###################################################
706 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
707 echo "Verify Lustre filesystem is up and running"
708 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
715 echo "Failing 2 CLIENTS"
719 echo "Test Lustre stability after CLIENT failure"
723 echo "Reintegrating CLIENTS"
724 reintegrate_clients || return 1
726 clients_up || return 3
727 sleep 2 # give it a little time for fully recovered before next test
731 clients_up || return 4
733 run_test 13 "Thirteen Failure Mode: MDS0,MDS1/CLIENTS/OST0,OST1 `date`"
734 ###################################################
737 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
738 echo "Verify Lustre filesystem is up and running"
739 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
746 echo "Failing 2 CLIENTS"
750 echo "Test Lustre stability after CLIENT failure"
754 echo "Reintegrating CLIENTS"
755 reintegrate_clients || return 1
757 clients_up || return 3
758 sleep 2 # give it a little time for fully recovered before next test
762 clients_up || return 4
764 run_test 14 "Fourteen Failure Mode: OST0,OST1/CLIENTS/MDS0,MDS1 `date`"
765 ###################################################
768 #Run availability after all failures
769 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
770 LOADTEST=${LOADTEST:-metadata-load.py}
771 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
773 run_test 15 "Running Availability for 6 hours..."
776 check_and_cleanup_lustre