X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Finsanity.sh;h=f0cb4036522c02dbfdf4cb6e9a8c28a156a92687;hb=6fcbb85b3a9a618e8dd7ee14c9fc7da7d23dab5d;hp=61a18f622c6fe56a9608d2612d85c3d6326f42e3;hpb=a590697063c0ef8a2cf9bd7f5e51cb757e166363;p=fs%2Flustre-release.git diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh index 61a18f6..f0cb403 100755 --- a/lustre/tests/insanity.sh +++ b/lustre/tests/insanity.sh @@ -9,14 +9,13 @@ LUSTRE=${LUSTRE:-`dirname $0`/..} init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging # ALWAYS_EXCEPT="10 $INSANITY_EXCEPT" if [ "$FAILURE_MODE" = "HARD" ]; then - mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \ - echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \ - echo "Except the tests: $CONFIG_EXCEPTIONS" && \ - ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS" + skip_env "$0: is not functional with FAILURE_MODE = HARD, please use recovery-double-scale, bz20407" + exit 0 fi # @@ -35,13 +34,14 @@ assert_env mds_HOST MDS_MKFS_OPTS assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT assert_env LIVE_CLIENT FSNAME -remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0 -remote_ost_nodsh && skip "remote OST with nodsh" && exit 0 +require_dsh_mds || exit 0 +require_dsh_ost || exit 0 # FAIL_CLIENTS list should not contain the LIVE_CLIENT FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g") DIR=${DIR:-$MOUNT} +TESTDIR=$DIR/d0.$(basename $0 .sh) ##### # fail clients round robin @@ -60,26 +60,6 @@ set_fail_client() { echo "fail $FAIL_CLIENT, next is $FAIL_NEXT" } -shutdown_client() { - client=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_DOWN $client - while ping -w 3 -c 1 $client > /dev/null 2>&1; do - echo "waiting for node $client to fail" - sleep 1 - done - elif [ "$FAILURE_MODE" = SOFT ]; then - zconf_umount $client $MOUNT -f - fi -} - -reboot_node() { - NODE=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_UP $NODE - fi -} - fail_clients() { num=$1 @@ -105,7 +85,7 @@ fail_clients() { echo "down clients: $DOWN_CLIENTS" for client in $DOWN_CLIENTS; do - reboot_node $client + boot_node $client done DOWN_NUM=`echo $DOWN_CLIENTS | wc -w` client_rmdirs @@ -131,29 +111,28 @@ client_touch() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do if echo $DOWN_CLIENTS | grep -q $c; then continue; fi - $PDSH $c touch $MOUNT/${c}_$file || return 1 + $PDSH $c touch $TESTDIR/${c}_$file || return 1 done } client_rm() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do - $PDSH $c rm $MOUNT/${c}_$file + $PDSH $c rm $TESTDIR/${c}_$file done } client_mkdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "$c mkdir $MOUNT/$c" - $PDSH $c "mkdir $MOUNT/$c" - $PDSH $c "ls -l $MOUNT/$c" + echo "$c mkdir $TESTDIR/$c" + $PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c" done } client_rmdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "rmdir $MOUNT/$c" - $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c" + echo "rmdir $TESTDIR/$c" + $PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c" done } @@ -164,18 +143,17 @@ clients_recover_osts() { check_and_setup_lustre +rm -rf $TESTDIR +mkdir -p $TESTDIR + # 9 Different Failure Modes Combinations echo "Starting Test 17 at `date`" test_0() { - facet_failover $SINGLEMDS - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 1; } + fail $SINGLEMDS for i in $(seq $OSTCOUNT) ; do - facet_failover ost$i || return 4 - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 3; } + fail ost$i done return 0 } @@ -194,7 +172,7 @@ test_2() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up shutdown_facet $SINGLEMDS reboot_facet $SINGLEMDS @@ -203,7 +181,7 @@ test_2() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPID=$! sleep 5 @@ -211,17 +189,17 @@ test_2() { echo "Reintegrating OST" reboot_facet ost1 - wait_for ost1 + wait_for_facet ost1 start_ost 1 || return 2 - wait_for $SINGLEMDS + wait_for_facet $SINGLEMDS start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $? #Check FS wait $DFPID clients_recover_osts ost1 echo "Verify reintegration" - client_df || return 1 + clients_up || return 1 } run_test 2 "Second Failure Mode: MDS/OST `date`" @@ -235,12 +213,11 @@ test_3() { [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" #MDS Portion - facet_failover $SINGLEMDS - wait $DFPID || echo df failed: $? + fail $SINGLEMDS #Check FS echo "Test Lustre stability after MDS failover" - client_df + clients_up #CLIENT Portion echo "Failing 2 CLIENTS" @@ -248,13 +225,13 @@ test_3() { #Check FS echo "Test Lustre stability after CLIENT failure" - client_df + clients_up #Reintegration echo "Reintegrating CLIENTS" reintegrate_clients || return 1 - client_df || return 3 + clients_up || return 3 sleep 2 # give it a little time for fully recovered before next test } run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`" @@ -269,7 +246,7 @@ test_4() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -281,17 +258,17 @@ test_4() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPIDB=$! sleep 5 #Reintegration echo "Reintegrating OST" reboot_facet ost1 - wait_for ost1 + wait_for_facet ost1 start_ost 1 - - wait_for $SINGLEMDS + + wait_for_facet $SINGLEMDS start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS #Check FS @@ -299,14 +276,14 @@ test_4() { wait $DFPIDB clients_recover_osts ost1 echo "Test Lustre stability after MDS failover" - client_df || return 1 + clients_up || return 1 } run_test 4 "Fourth Failure Mode: OST/MDS `date`" ################################################### ############### Fifth Failure Mode ############### test_5() { - [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0 + [ $OSTCOUNT -lt 2 ] && skip_env "$OSTCOUNT < 2, not enough OSTs" && return 0 echo "Fifth Failure Mode: OST/OST `date`" @@ -314,7 +291,7 @@ test_5() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up #OST Portion shutdown_facet ost1 @@ -322,7 +299,7 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -332,15 +309,15 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDB=$! sleep 5 #Reintegration echo "Reintegrating OSTs" - wait_for ost1 + wait_for_facet ost1 start_ost 1 - wait_for ost2 + wait_for_facet ost2 start_ost 2 clients_recover_osts ost1 @@ -349,7 +326,7 @@ test_5() { wait $DFPIDA wait $DFPIDB - client_df || return 2 + clients_up || return 2 } run_test 5 "Fifth Failure Mode: OST/OST `date`" ################################################### @@ -362,7 +339,7 @@ test_6() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 2 #OST Portion @@ -371,7 +348,7 @@ test_6() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! echo DFPIDA=$DFPIDA sleep 5 @@ -382,25 +359,25 @@ test_6() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df & + clients_up & DFPIDB=$! echo DFPIDB=$DFPIDB sleep 5 #Reintegration echo "Reintegrating OST/CLIENTs" - wait_for ost1 + wait_for_facet ost1 start_ost 1 reintegrate_clients || return 1 sleep 5 - wait_remote_prog df $((TIMEOUT * 3 + 10)) + wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20)) wait $DFPIDA wait $DFPIDB echo "Verifying mount" [ -z "$(mounted_lustre_filesystems)" ] && return 3 - client_df + clients_up } run_test 6 "Sixth Failure Mode: OST/CLIENT `date`" ################################################### @@ -414,7 +391,7 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -423,9 +400,9 @@ test_7() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -435,22 +412,19 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && return 2 - client_df + clients_up client_rm testfile #MDS Portion - facet_failover $SINGLEMDS + fail $SINGLEMDS - #Check FS - echo "Test Lustre stability after MDS failover" - wait $DFPID || echo "df on down clients fails " || return 1 - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs" reintegrate_clients || return 2 - client_df + clients_up #Sleep echo "wait 1 minutes" @@ -468,7 +442,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile #CLIENT Portion @@ -477,9 +451,9 @@ test_8() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -489,7 +463,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile @@ -499,20 +473,20 @@ test_8() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPID=$! sleep 5 #non-failout hangs forever here - #$PDSH $LIVE_CLIENT "ls -l $MOUNT" - #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + #$PDSH $LIVE_CLIENT "ls -l $TESTDIR" + #$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs/OST" reintegrate_clients || return 3 - wait_for ost1 + wait_for_facet ost1 start_ost 1 wait $DFPID - client_df || return 1 + clients_up || return 1 client_touch testfile2 || return 2 #Sleep @@ -531,7 +505,7 @@ test_9() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -540,9 +514,9 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2 #Sleep echo "Wait 1 minutes" @@ -550,7 +524,7 @@ test_9() { #Create files echo "Verify Lustre filesystem is up and running" - $PDSH $LIVE_CLIENT df $MOUNT || return 3 + client_up $LIVE_CLIENT || return 3 client_touch testfile || return 4 #CLIENT Portion @@ -559,14 +533,14 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6 #Reintegration echo "Reintegrating CLIENTs/CLIENTs" reintegrate_clients || return 7 - client_df + clients_up #Sleep echo "Wait 1 minutes" @@ -583,6 +557,6 @@ test_10() { } run_test 10 "Running Availability for 6 hours..." -equals_msg `basename $0`: test complete, cleaning up +complete $(basename $0) $SECONDS check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +exit_status