X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Finsanity.sh;h=f0cb4036522c02dbfdf4cb6e9a8c28a156a92687;hb=fa507031d245b08c7f24efed32819daf2aa42ab3;hp=7e2c57ab645611bfbed7c11919bb4987ec34c32f;hpb=7618f44f52222843feeb37ddf3d91a17bef5f256;p=fs%2Flustre-release.git diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh index 7e2c57a..f0cb403 100755 --- a/lustre/tests/insanity.sh +++ b/lustre/tests/insanity.sh @@ -9,14 +9,13 @@ LUSTRE=${LUSTRE:-`dirname $0`/..} init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -# 13129 13129 -ALWAYS_EXCEPT="2 4 10 $INSANITY_EXCEPT" +init_logging +# +ALWAYS_EXCEPT="10 $INSANITY_EXCEPT" if [ "$FAILURE_MODE" = "HARD" ]; then - mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \ - echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \ - echo "Except the tests: $CONFIG_EXCEPTIONS" && \ - ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS" + skip_env "$0: is not functional with FAILURE_MODE = HARD, please use recovery-double-scale, bz20407" + exit 0 fi # @@ -35,11 +34,14 @@ assert_env mds_HOST MDS_MKFS_OPTS assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT assert_env LIVE_CLIENT FSNAME +require_dsh_mds || exit 0 +require_dsh_ost || exit 0 -# This can be a regexp, to allow more clients -CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS`"} +# FAIL_CLIENTS list should not contain the LIVE_CLIENT +FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g") DIR=${DIR:-$MOUNT} +TESTDIR=$DIR/d0.$(basename $0 .sh) ##### # fail clients round robin @@ -58,26 +60,6 @@ set_fail_client() { echo "fail $FAIL_CLIENT, next is $FAIL_NEXT" } -shutdown_client() { - client=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_DOWN $client - while ping -w 3 -c 1 $client > /dev/null 2>&1; do - echo "waiting for node $client to fail" - sleep 1 - done - elif [ "$FAILURE_MODE" = SOFT ]; then - zconf_umount $client $MOUNT -f - fi -} - -reboot_node() { - NODE=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_UP $NODE - fi -} - fail_clients() { num=$1 @@ -103,7 +85,7 @@ fail_clients() { echo "down clients: $DOWN_CLIENTS" for client in $DOWN_CLIENTS; do - reboot_node $client + boot_node $client done DOWN_NUM=`echo $DOWN_CLIENTS | wc -w` client_rmdirs @@ -129,29 +111,28 @@ client_touch() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do if echo $DOWN_CLIENTS | grep -q $c; then continue; fi - $PDSH $c touch $MOUNT/${c}_$file || return 1 + $PDSH $c touch $TESTDIR/${c}_$file || return 1 done } client_rm() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do - $PDSH $c rm $MOUNT/${c}_$file + $PDSH $c rm $TESTDIR/${c}_$file done } client_mkdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "$c mkdir $MOUNT/$c" - $PDSH $c "mkdir $MOUNT/$c" - $PDSH $c "ls -l $MOUNT/$c" + echo "$c mkdir $TESTDIR/$c" + $PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c" done } client_rmdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "rmdir $MOUNT/$c" - $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c" + echo "rmdir $TESTDIR/$c" + $PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c" done } @@ -160,20 +141,19 @@ clients_recover_osts() { # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover" } -cleanup_and_setup_lustre +check_and_setup_lustre + +rm -rf $TESTDIR +mkdir -p $TESTDIR # 9 Different Failure Modes Combinations echo "Starting Test 17 at `date`" test_0() { - facet_failover $SINGLEMDS - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 1; } + fail $SINGLEMDS for i in $(seq $OSTCOUNT) ; do - facet_failover ost$i || return 4 - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 3; } + fail ost$i done return 0 } @@ -192,7 +172,7 @@ test_2() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up shutdown_facet $SINGLEMDS reboot_facet $SINGLEMDS @@ -201,7 +181,7 @@ test_2() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPID=$! sleep 5 @@ -209,17 +189,17 @@ test_2() { echo "Reintegrating OST" reboot_facet ost1 - wait_for ost1 + wait_for_facet ost1 start_ost 1 || return 2 - wait_for $SINGLEMDS + wait_for_facet $SINGLEMDS start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $? #Check FS wait $DFPID clients_recover_osts ost1 echo "Verify reintegration" - client_df || return 1 + clients_up || return 1 } run_test 2 "Second Failure Mode: MDS/OST `date`" @@ -233,12 +213,11 @@ test_3() { [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" #MDS Portion - facet_failover $SINGLEMDS - wait $DFPID || echo df failed: $? + fail $SINGLEMDS #Check FS echo "Test Lustre stability after MDS failover" - client_df + clients_up #CLIENT Portion echo "Failing 2 CLIENTS" @@ -246,13 +225,13 @@ test_3() { #Check FS echo "Test Lustre stability after CLIENT failure" - client_df + clients_up #Reintegration echo "Reintegrating CLIENTS" reintegrate_clients || return 1 - client_df || return 3 + clients_up || return 3 sleep 2 # give it a little time for fully recovered before next test } run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`" @@ -267,7 +246,7 @@ test_4() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -279,17 +258,17 @@ test_4() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPIDB=$! sleep 5 #Reintegration echo "Reintegrating OST" reboot_facet ost1 - wait_for ost1 + wait_for_facet ost1 start_ost 1 - - wait_for $SINGLEMDS + + wait_for_facet $SINGLEMDS start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS #Check FS @@ -297,14 +276,14 @@ test_4() { wait $DFPIDB clients_recover_osts ost1 echo "Test Lustre stability after MDS failover" - client_df || return 1 + clients_up || return 1 } run_test 4 "Fourth Failure Mode: OST/MDS `date`" ################################################### ############### Fifth Failure Mode ############### test_5() { - [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0 + [ $OSTCOUNT -lt 2 ] && skip_env "$OSTCOUNT < 2, not enough OSTs" && return 0 echo "Fifth Failure Mode: OST/OST `date`" @@ -312,7 +291,7 @@ test_5() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up #OST Portion shutdown_facet ost1 @@ -320,7 +299,7 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -330,15 +309,15 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDB=$! sleep 5 #Reintegration echo "Reintegrating OSTs" - wait_for ost1 + wait_for_facet ost1 start_ost 1 - wait_for ost2 + wait_for_facet ost2 start_ost 2 clients_recover_osts ost1 @@ -347,7 +326,7 @@ test_5() { wait $DFPIDA wait $DFPIDB - client_df || return 2 + clients_up || return 2 } run_test 5 "Fifth Failure Mode: OST/OST `date`" ################################################### @@ -360,7 +339,7 @@ test_6() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 2 #OST Portion @@ -369,8 +348,9 @@ test_6() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! + echo DFPIDA=$DFPIDA sleep 5 #CLIENT Portion @@ -379,22 +359,25 @@ test_6() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df & + clients_up & DFPIDB=$! + echo DFPIDB=$DFPIDB sleep 5 #Reintegration echo "Reintegrating OST/CLIENTs" - wait_for ost1 + wait_for_facet ost1 start_ost 1 - reintegrate_clients + reintegrate_clients || return 1 sleep 5 + wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20)) wait $DFPIDA wait $DFPIDB + echo "Verifying mount" [ -z "$(mounted_lustre_filesystems)" ] && return 3 - client_df + clients_up } run_test 6 "Sixth Failure Mode: OST/CLIENT `date`" ################################################### @@ -408,7 +391,7 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -417,9 +400,9 @@ test_7() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -429,22 +412,19 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && return 2 - client_df + clients_up client_rm testfile #MDS Portion - facet_failover $SINGLEMDS + fail $SINGLEMDS - #Check FS - echo "Test Lustre stability after MDS failover" - wait $DFPID || echo "df on down clients fails " || return 1 - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs" reintegrate_clients || return 2 - client_df + clients_up #Sleep echo "wait 1 minutes" @@ -462,7 +442,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile #CLIENT Portion @@ -471,9 +451,9 @@ test_8() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -483,7 +463,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile @@ -493,20 +473,20 @@ test_8() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPID=$! sleep 5 #non-failout hangs forever here - #$PDSH $LIVE_CLIENT "ls -l $MOUNT" - #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + #$PDSH $LIVE_CLIENT "ls -l $TESTDIR" + #$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs/OST" reintegrate_clients || return 3 - wait_for ost1 + wait_for_facet ost1 start_ost 1 wait $DFPID - client_df || return 1 + clients_up || return 1 client_touch testfile2 || return 2 #Sleep @@ -525,7 +505,7 @@ test_9() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -534,9 +514,9 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2 #Sleep echo "Wait 1 minutes" @@ -544,7 +524,7 @@ test_9() { #Create files echo "Verify Lustre filesystem is up and running" - $PDSH $LIVE_CLIENT df $MOUNT || return 3 + client_up $LIVE_CLIENT || return 3 client_touch testfile || return 4 #CLIENT Portion @@ -553,14 +533,14 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6 #Reintegration echo "Reintegrating CLIENTs/CLIENTs" reintegrate_clients || return 7 - client_df + clients_up #Sleep echo "Wait 1 minutes" @@ -577,6 +557,6 @@ test_10() { } run_test 10 "Running Availability for 6 hours..." -equals_msg `basename $0`: test complete, cleaning up +complete $(basename $0) $SECONDS check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +exit_status