2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
13 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
15 if [ "$FAILURE_MODE" = "HARD" ]; then
16 mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \
17 echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \
18 echo "Except the tests: $CONFIG_EXCEPTIONS" && \
19 ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
23 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
26 CLEANUP=${CLEANUP:-""}
30 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
31 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
32 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
34 assert_env mds_HOST MDS_MKFS_OPTS
35 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
36 assert_env LIVE_CLIENT FSNAME
38 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
39 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
44 # fail clients round robin
46 # list of failable clients
47 FAIL_LIST=($FAIL_CLIENTS)
48 FAIL_NUM=${#FAIL_LIST[*]}
51 DOWN_NUM=0 # number of nodes currently down
53 # set next client to fail
55 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
56 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
57 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
62 if [ "$FAILURE_MODE" = HARD ]; then
64 while ping -w 3 -c 1 $client > /dev/null 2>&1; do
65 echo "waiting for node $client to fail"
68 elif [ "$FAILURE_MODE" = SOFT ]; then
69 zconf_umount $client $MOUNT -f
75 if [ "$FAILURE_MODE" = HARD ]; then
83 log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
84 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
85 num=$((FAIL_NUM - DOWN_NUM))
88 if [ -z "$num" ] || [ "$num" -le 0 ]; then
89 log "No clients failed!"
95 for i in `seq $num`; do
98 DOWN_CLIENTS="$DOWN_CLIENTS $client"
99 shutdown_client $client
102 echo "down clients: $DOWN_CLIENTS"
104 for client in $DOWN_CLIENTS; do
107 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
111 reintegrate_clients() {
112 for client in $DOWN_CLIENTS; do
113 wait_for_host $client
114 echo "Restarting $client"
115 zconf_mount $client $MOUNT || return 1
122 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
129 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
130 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
131 $PDSH $c touch $MOUNT/${c}_$file || return 1
137 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
138 $PDSH $c rm $MOUNT/${c}_$file
143 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
144 echo "$c mkdir $MOUNT/$c"
145 $PDSH $c "mkdir $MOUNT/$c"
146 $PDSH $c "ls -l $MOUNT/$c"
151 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
152 echo "rmdir $MOUNT/$c"
153 $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c"
157 clients_recover_osts() {
159 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
162 cleanup_and_setup_lustre
164 # 9 Different Failure Modes Combinations
165 echo "Starting Test 17 at `date`"
168 facet_failover $SINGLEMDS
169 echo "Waiting for df pid: $DFPID"
170 wait $DFPID || { echo "df returned $?" && return 1; }
172 for i in $(seq $OSTCOUNT) ; do
173 facet_failover ost$i || return 4
174 echo "Waiting for df pid: $DFPID"
175 wait $DFPID || { echo "df returned $?" && return 3; }
179 run_test 0 "Fail all nodes, independently"
181 ############### First Failure Mode ###############
183 echo "Don't do a MDS - MDS Failure Case"
184 echo "This makes no sense"
186 run_test 1 "MDS/MDS failure"
187 ###################################################
189 ############### Second Failure Mode ###############
191 echo "Verify Lustre filesystem is up and running"
192 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
196 shutdown_facet $SINGLEMDS
197 reboot_facet $SINGLEMDS
199 # prepare for MDS failover
200 change_active $SINGLEMDS
201 reboot_facet $SINGLEMDS
209 echo "Reintegrating OST"
212 start_ost 1 || return 2
215 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
219 clients_recover_osts ost1
220 echo "Verify reintegration"
221 client_df || return 1
224 run_test 2 "Second Failure Mode: MDS/OST `date`"
225 ###################################################
228 ############### Third Failure Mode ###############
231 echo "Verify Lustre filesystem is up and running"
232 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
235 facet_failover $SINGLEMDS
236 wait $DFPID || echo df failed: $?
239 echo "Test Lustre stability after MDS failover"
243 echo "Failing 2 CLIENTS"
247 echo "Test Lustre stability after CLIENT failure"
251 echo "Reintegrating CLIENTS"
252 reintegrate_clients || return 1
254 client_df || return 3
255 sleep 2 # give it a little time for fully recovered before next test
257 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
258 ###################################################
260 ############### Fourth Failure Mode ###############
262 echo "Fourth Failure Mode: OST/MDS `date`"
268 echo "Test Lustre stability after OST failure"
274 shutdown_facet $SINGLEMDS
275 reboot_facet $SINGLEMDS
277 # prepare for MDS failover
278 change_active $SINGLEMDS
279 reboot_facet $SINGLEMDS
286 echo "Reintegrating OST"
292 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
297 clients_recover_osts ost1
298 echo "Test Lustre stability after MDS failover"
299 client_df || return 1
301 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
302 ###################################################
304 ############### Fifth Failure Mode ###############
306 [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0
308 echo "Fifth Failure Mode: OST/OST `date`"
311 echo "Verify Lustre filesystem is up and running"
312 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
321 echo "Test Lustre stability after OST failure"
331 echo "Test Lustre stability after OST failure"
337 echo "Reintegrating OSTs"
343 clients_recover_osts ost1
344 clients_recover_osts ost2
349 client_df || return 2
351 run_test 5 "Fifth Failure Mode: OST/OST `date`"
352 ###################################################
354 ############### Sixth Failure Mode ###############
356 echo "Sixth Failure Mode: OST/CLIENT `date`"
359 echo "Verify Lustre filesystem is up and running"
360 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
363 client_touch testfile || return 2
370 echo "Test Lustre stability after OST failure"
377 echo "Failing CLIENTs"
381 echo "Test Lustre stability after CLIENTs failure"
388 echo "Reintegrating OST/CLIENTs"
391 reintegrate_clients || return 1
394 wait_remote_prog df $((TIMEOUT * 3 + 10))
398 echo "Verifying mount"
399 [ -z "$(mounted_lustre_filesystems)" ] && return 3
402 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
403 ###################################################
406 ############### Seventh Failure Mode ###############
408 echo "Seventh Failure Mode: CLIENT/MDS `date`"
411 echo "Verify Lustre filesystem is up and running"
412 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
415 client_touch testfile || return 1
418 echo "Part 1: Failing CLIENT"
422 echo "Test Lustre stability after CLIENTs failure"
424 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
425 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
428 echo "Wait 1 minutes"
432 echo "Verify Lustre filesystem is up and running"
433 [ -z "$(mounted_lustre_filesystems)" ] && return 2
439 facet_failover $SINGLEMDS
442 echo "Test Lustre stability after MDS failover"
443 wait $DFPID || echo "df on down clients fails " || return 1
444 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
445 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
448 echo "Reintegrating CLIENTs"
449 reintegrate_clients || return 2
453 echo "wait 1 minutes"
456 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
457 ###################################################
460 ############### Eighth Failure Mode ###############
462 echo "Eighth Failure Mode: CLIENT/OST `date`"
465 echo "Verify Lustre filesystem is up and running"
466 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
469 client_touch testfile
472 echo "Failing CLIENTs"
476 echo "Test Lustre stability after CLIENTs failure"
478 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
479 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
482 echo "Wait 1 minutes"
486 echo "Verify Lustre filesystem is up and running"
487 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
490 client_touch testfile
498 echo "Test Lustre stability after OST failure"
502 #non-failout hangs forever here
503 #$PDSH $LIVE_CLIENT "ls -l $MOUNT"
504 #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
507 echo "Reintegrating CLIENTs/OST"
508 reintegrate_clients || return 3
512 client_df || return 1
513 client_touch testfile2 || return 2
516 echo "Wait 1 minutes"
519 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
520 ###################################################
523 ############### Ninth Failure Mode ###############
528 echo "Verify Lustre filesystem is up and running"
529 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
532 client_touch testfile || return 1
535 echo "Failing CLIENTs"
539 echo "Test Lustre stability after CLIENTs failure"
541 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
542 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
545 echo "Wait 1 minutes"
549 echo "Verify Lustre filesystem is up and running"
550 $PDSH $LIVE_CLIENT df $MOUNT || return 3
551 client_touch testfile || return 4
554 echo "Failing CLIENTs"
558 echo "Test Lustre stability after CLIENTs failure"
560 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
561 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
564 echo "Reintegrating CLIENTs/CLIENTs"
565 reintegrate_clients || return 7
569 echo "Wait 1 minutes"
572 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
573 ###################################################
576 #Run availability after all failures
577 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
578 LOADTEST=${LOADTEST:-metadata-load.py}
579 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
581 run_test 10 "Running Availability for 6 hours..."
583 equals_msg `basename $0`: test complete, cleaning up
584 check_and_cleanup_lustre
585 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true