2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
14 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
16 if [ "$FAILURE_MODE" = "HARD" ]; then
17 skip_env "$0: is not functional with FAILURE_MODE = HARD, please use recovery-double-scale, bz20407"
22 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
25 CLEANUP=${CLEANUP:-""}
29 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
30 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
31 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
33 assert_env mds_HOST MDS_MKFS_OPTS
34 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
35 assert_env LIVE_CLIENT FSNAME
37 require_dsh_mds || exit 0
38 require_dsh_ost || exit 0
40 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
41 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
44 TESTDIR=$DIR/d0.$(basename $0 .sh)
47 # fail clients round robin
49 # list of failable clients
50 FAIL_LIST=($FAIL_CLIENTS)
51 FAIL_NUM=${#FAIL_LIST[*]}
54 DOWN_NUM=0 # number of nodes currently down
56 # set next client to fail
58 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
59 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
60 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
66 log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
67 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
68 num=$((FAIL_NUM - DOWN_NUM))
71 if [ -z "$num" ] || [ "$num" -le 0 ]; then
72 log "No clients failed!"
78 for i in `seq $num`; do
81 DOWN_CLIENTS="$DOWN_CLIENTS $client"
82 shutdown_client $client
85 echo "down clients: $DOWN_CLIENTS"
87 for client in $DOWN_CLIENTS; do
90 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
94 reintegrate_clients() {
95 for client in $DOWN_CLIENTS; do
97 echo "Restarting $client"
98 zconf_mount $client $MOUNT || return 1
105 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
112 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
113 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
114 $PDSH $c touch $TESTDIR/${c}_$file || return 1
120 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
121 $PDSH $c rm $TESTDIR/${c}_$file
126 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
127 echo "$c mkdir $TESTDIR/$c"
128 $PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c"
133 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
134 echo "rmdir $TESTDIR/$c"
135 $PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c"
139 clients_recover_osts() {
141 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
144 check_and_setup_lustre
149 # 9 Different Failure Modes Combinations
150 echo "Starting Test 17 at `date`"
155 for i in $(seq $OSTCOUNT) ; do
160 run_test 0 "Fail all nodes, independently"
162 ############### First Failure Mode ###############
164 echo "Don't do a MDS - MDS Failure Case"
165 echo "This makes no sense"
167 run_test 1 "MDS/MDS failure"
168 ###################################################
170 ############### Second Failure Mode ###############
172 echo "Verify Lustre filesystem is up and running"
173 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
177 shutdown_facet $SINGLEMDS
178 reboot_facet $SINGLEMDS
180 # prepare for MDS failover
181 change_active $SINGLEMDS
182 reboot_facet $SINGLEMDS
190 echo "Reintegrating OST"
193 start_ost 1 || return 2
195 wait_for_facet $SINGLEMDS
196 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
200 clients_recover_osts ost1
201 echo "Verify reintegration"
202 clients_up || return 1
205 run_test 2 "Second Failure Mode: MDS/OST `date`"
206 ###################################################
209 ############### Third Failure Mode ###############
212 echo "Verify Lustre filesystem is up and running"
213 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
219 echo "Test Lustre stability after MDS failover"
223 echo "Failing 2 CLIENTS"
227 echo "Test Lustre stability after CLIENT failure"
231 echo "Reintegrating CLIENTS"
232 reintegrate_clients || return 1
234 clients_up || return 3
235 sleep 2 # give it a little time for fully recovered before next test
237 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
238 ###################################################
240 ############### Fourth Failure Mode ###############
242 echo "Fourth Failure Mode: OST/MDS `date`"
248 echo "Test Lustre stability after OST failure"
254 shutdown_facet $SINGLEMDS
255 reboot_facet $SINGLEMDS
257 # prepare for MDS failover
258 change_active $SINGLEMDS
259 reboot_facet $SINGLEMDS
266 echo "Reintegrating OST"
271 wait_for_facet $SINGLEMDS
272 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
277 clients_recover_osts ost1
278 echo "Test Lustre stability after MDS failover"
279 clients_up || return 1
281 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
282 ###################################################
284 ############### Fifth Failure Mode ###############
286 [ $OSTCOUNT -lt 2 ] && skip_env "$OSTCOUNT < 2, not enough OSTs" && return 0
288 echo "Fifth Failure Mode: OST/OST `date`"
291 echo "Verify Lustre filesystem is up and running"
292 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
301 echo "Test Lustre stability after OST failure"
311 echo "Test Lustre stability after OST failure"
317 echo "Reintegrating OSTs"
323 clients_recover_osts ost1
324 clients_recover_osts ost2
329 clients_up || return 2
331 run_test 5 "Fifth Failure Mode: OST/OST `date`"
332 ###################################################
334 ############### Sixth Failure Mode ###############
336 echo "Sixth Failure Mode: OST/CLIENT `date`"
339 echo "Verify Lustre filesystem is up and running"
340 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
343 client_touch testfile || return 2
350 echo "Test Lustre stability after OST failure"
357 echo "Failing CLIENTs"
361 echo "Test Lustre stability after CLIENTs failure"
368 echo "Reintegrating OST/CLIENTs"
371 reintegrate_clients || return 1
374 wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20))
378 echo "Verifying mount"
379 [ -z "$(mounted_lustre_filesystems)" ] && return 3
382 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
383 ###################################################
386 ############### Seventh Failure Mode ###############
388 echo "Seventh Failure Mode: CLIENT/MDS `date`"
391 echo "Verify Lustre filesystem is up and running"
392 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
395 client_touch testfile || return 1
398 echo "Part 1: Failing CLIENT"
402 echo "Test Lustre stability after CLIENTs failure"
404 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
405 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
408 echo "Wait 1 minutes"
412 echo "Verify Lustre filesystem is up and running"
413 [ -z "$(mounted_lustre_filesystems)" ] && return 2
421 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
422 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
425 echo "Reintegrating CLIENTs"
426 reintegrate_clients || return 2
430 echo "wait 1 minutes"
433 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
434 ###################################################
437 ############### Eighth Failure Mode ###############
439 echo "Eighth Failure Mode: CLIENT/OST `date`"
442 echo "Verify Lustre filesystem is up and running"
443 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
446 client_touch testfile
449 echo "Failing CLIENTs"
453 echo "Test Lustre stability after CLIENTs failure"
455 $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
456 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
459 echo "Wait 1 minutes"
463 echo "Verify Lustre filesystem is up and running"
464 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
467 client_touch testfile
475 echo "Test Lustre stability after OST failure"
479 #non-failout hangs forever here
480 #$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
481 #$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
484 echo "Reintegrating CLIENTs/OST"
485 reintegrate_clients || return 3
489 clients_up || return 1
490 client_touch testfile2 || return 2
493 echo "Wait 1 minutes"
496 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
497 ###################################################
500 ############### Ninth Failure Mode ###############
505 echo "Verify Lustre filesystem is up and running"
506 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
509 client_touch testfile || return 1
512 echo "Failing CLIENTs"
516 echo "Test Lustre stability after CLIENTs failure"
518 $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1
519 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2
522 echo "Wait 1 minutes"
526 echo "Verify Lustre filesystem is up and running"
527 client_up $LIVE_CLIENT || return 3
528 client_touch testfile || return 4
531 echo "Failing CLIENTs"
535 echo "Test Lustre stability after CLIENTs failure"
537 $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5
538 $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6
541 echo "Reintegrating CLIENTs/CLIENTs"
542 reintegrate_clients || return 7
546 echo "Wait 1 minutes"
549 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
550 ###################################################
553 #Run availability after all failures
554 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
555 LOADTEST=${LOADTEST:-metadata-load.py}
556 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
558 run_test 10 "Running Availability for 6 hours..."
560 equals_msg `basename $0`: test complete, cleaning up
561 check_and_cleanup_lustre
562 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true