X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Freplay-single.sh;h=4ccfe84e8e860a76d7ee766b8b40b56f38aa3221;hp=209206e806d5ea6468ceb0e4b3e8f4221d9e71c0;hb=4ee47d0f475cf42f9796f13e78c8cfb1e3c94807;hpb=1b7a0af5e3f4e88e845fbcd47c72135aed8f5369 diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 209206e..4ccfe84 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -16,10 +16,11 @@ init_test_env $@ CHECK_GRANT=${CHECK_GRANT:-"yes"} GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""} +remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0 # Skip these tests -# bug number: 17466 -ALWAYS_EXCEPT="61d $REPLAY_SINGLE_EXCEPT" +# bug number: 17466 15962 +ALWAYS_EXCEPT="61d 33b $REPLAY_SINGLE_EXCEPT" if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then CONFIG_EXCEPTIONS="0b 42 47 61a 61c" @@ -33,7 +34,7 @@ fi build_test_filter -cleanup_and_setup_lustre +check_and_setup_lustre mkdir -p $DIR @@ -50,6 +51,8 @@ test_0a() { # was test_0 run_test 0a "empty replay" test_0b() { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + # this test attempts to trigger a race in the precreation code, # and must run before any other objects are created on the filesystem fail ost1 @@ -154,12 +157,13 @@ test_2b() { run_test 2b "touch" test_3a() { + local file=$DIR/$tfile replay_barrier $SINGLEMDS - mcreate $DIR/$tfile - o_directory $DIR/$tfile + mcreate $file + openfile -f O_DIRECTORY $file fail $SINGLEMDS - $CHECKSTAT -t file $DIR/$tfile || return 2 - rm $DIR/$tfile + $CHECKSTAT -t file $file || return 2 + rm $file } run_test 3a "replay failed open(O_DIRECTORY)" @@ -217,7 +221,7 @@ test_5() { done fail $SINGLEMDS for i in `seq 220`; do - grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i" + grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i" done rm -rf $DIR/$tfile-* sleep 3 @@ -456,6 +460,8 @@ test_20a() { # was test_20 run_test 20a "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)" test_20b() { # bug 10480 + # XXX increase the debug level temporary + do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug=0x33f0406; $LCTL set_param debug_mb=150" BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'` dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 & @@ -471,7 +477,7 @@ test_20b() { # bug 10480 fail $SINGLEMDS # start orphan recovery df -P $DIR || df -P $DIR || true # reconnect - wait_mds_recovery_done || error "MDS recovery not done" + wait_recovery_complete $SINGLEMDS || error "MDS recovery not done" # FIXME just because recovery is done doesn't mean we've finished # orphan cleanup. Fake it with a sleep for now... @@ -480,6 +486,8 @@ test_20b() { # bug 10480 log "before $BEFOREUSED, after $AFTERUSED" [ $AFTERUSED -gt $((BEFOREUSED + 20)) ] && \ error "after $AFTERUSED > before $BEFOREUSED" + # XXX decrease it back + do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug=$PTLDEBUG; $LCTL set_param debug_mb=$DEBUG_SIZE" return 0 } run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)" @@ -727,7 +735,7 @@ test_33a() { # was test_33 } run_test 33a "abort recovery before client does replay" -# Stale FID sequence +# Stale FID sequence bug 15962 test_33b() { # was test_33a replay_barrier $SINGLEMDS createmany -o $DIR/$tfile-%d 10 @@ -918,6 +926,8 @@ run_test 42 "recovery after ost failure" # timeout in MDS/OST recovery RPC will LBUG MDS test_43() { # bug 2530 + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + replay_barrier $SINGLEMDS # OBD_FAIL_OST_CREATE_NET 0x204 @@ -1009,6 +1019,8 @@ test_46() { run_test 46 "Don't leak file handle after open resend (3325)" test_47() { # bug 2824 + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + # create some files to make sure precreate has been done on all # OSTs. (just in case this test is run independently) createmany -o $DIR/$tfile 20 || return 1 @@ -1032,17 +1044,18 @@ test_47() { # bug 2824 run_test 47 "MDS->OSC failure during precreate cleanup (2824)" test_48() { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + [ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2 OSTs -- skipping" && return + replay_barrier $SINGLEMDS createmany -o $DIR/$tfile 20 || return 1 # OBD_FAIL_OST_EROFS 0x216 - fail $SINGLEMDS + facet_failover $SINGLEMDS do_facet ost1 "lctl set_param fail_loc=0x80000216" df $MOUNT || return 2 createmany -o $DIR/$tfile 20 20 || return 2 unlinkmany $DIR/$tfile 40 || return 3 - - do_facet ost1 "lctl set_param fail_loc=0" return 0 } run_test 48 "MDS->OSC failure during precreate cleanup (2824)" @@ -1104,6 +1117,8 @@ test_53a() { run_test 53a "|X| close request while two MDC requests in flight" test_53b() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1133,6 +1148,8 @@ test_53b() { run_test 53b "|X| open request while two MDC requests in flight" test_53c() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1164,6 +1181,8 @@ test_53c() { run_test 53c "|X| open request and close request while two MDC requests in flight" test_53d() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1171,8 +1190,8 @@ test_53d() { # give multiop a chance to open sleep 1 - #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f - do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f" + #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b + do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b" kill -USR1 $close_pid cancel_lru_locks mdc # force the close do_facet $SINGLEMDS "lctl set_param fail_loc=0" @@ -1190,6 +1209,8 @@ test_53d() { run_test 53d "|X| close reply while two MDC requests in flight" test_53e() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1219,6 +1240,8 @@ test_53e() { run_test 53e "|X| open reply while two MDC requests in flight" test_53f() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1230,8 +1253,8 @@ test_53f() { open_pid=$! sleep 1 - #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f - do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f" + #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b + do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b" kill -USR1 $close_pid cancel_lru_locks mdc # force the close @@ -1250,6 +1273,8 @@ test_53f() { run_test 53f "|X| open reply and close reply while two MDC requests in flight" test_53g() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1281,6 +1306,8 @@ test_53g() { run_test 53g "|X| drop open reply and close request while close and open are both in flight" test_53h() { + rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2 + mkdir -p $DIR/${tdir}-1 mkdir -p $DIR/${tdir}-2 multiop $DIR/${tdir}-1/f O_c & @@ -1292,8 +1319,8 @@ test_53h() { open_pid=$! sleep 1 - #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f - do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f" + #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b + do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b" kill -USR1 $close_pid cancel_lru_locks mdc # force the close sleep 1 @@ -1403,6 +1430,8 @@ run_test 58c "resend/reconstruct setxattr op" # log_commit_thread vs filter_destroy race used to lead to import use after free # bug 11658 test_59() { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + mkdir -p $DIR/$tdir createmany -o $DIR/$tdir/$tfile-%d 200 sync @@ -1417,21 +1446,6 @@ test_59() { } run_test 59 "test log_commit_thread vs filter_destroy race" -# bug 17323 -test_59b() { - mkdir -p $DIR/$tdir - createmany -o $DIR/$tdir/$tfile-%d 2000 - sync -#define OBD_FAIL_OBD_LOG_CANCEL_REP 0x606 - do_facet $SINGLEMDS "lctl set_param fail_loc=0x606" - unlinkmany $DIR/$tdir/$tfile-%d 2000 - sleep 60 - do_facet $SINGLEMDS "lctl set_param fail_loc=0x0" - do_facet $SINGLEMDS $LCTL dk | grep -q "RESENT cancel req" || return 1 - rmdir $DIR/$tdir -} -run_test 59b "resent handle in llog_origin_handle_cancel" - # race between add unlink llog vs cat log init in post_recovery (only for b1_6) # bug 12086: should no oops and No ctxt error for this test test_60() { @@ -1448,18 +1462,20 @@ run_test 60 "test llog post recovery init vs llog unlink" #test race llog recovery thread vs llog cleanup test_61a() { # was test_61 - mkdir $DIR/$tdir + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + + mkdir -p $DIR/$tdir createmany -o $DIR/$tdir/$tfile-%d 800 - replay_barrier ost1 -# OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221 - unlinkmany $DIR/$tdir/$tfile-%d 800 + replay_barrier ost1 +# OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221 + unlinkmany $DIR/$tdir/$tfile-%d 800 set_nodes_failloc "$(osts_nodes)" 0x80000221 facet_failover ost1 - sleep 10 + sleep 10 fail ost1 sleep 30 set_nodes_failloc "$(osts_nodes)" 0x0 - + $CHECKSTAT -t file $DIR/$tdir/$tfile-* && return 1 rmdir $DIR/$tdir } @@ -1467,9 +1483,9 @@ run_test 61a "test race llog recovery vs llog cleanup" #test race mds llog sync vs llog cleanup test_61b() { -# OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x140 - do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000140" - facet_failover $SINGLEMDS +# OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x13a + do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013a" + facet_failover $SINGLEMDS sleep 10 fail $SINGLEMDS do_facet client dd if=/dev/zero of=$DIR/$tfile bs=4k count=1 || return 1 @@ -1478,21 +1494,33 @@ run_test 61b "test race mds llog sync vs llog cleanup" #test race cancel cookie cb vs llog cleanup test_61c() { -# OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222 - touch $DIR/$tfile + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + +# OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222 + touch $DIR/$tfile set_nodes_failloc "$(osts_nodes)" 0x80000222 - rm $DIR/$tfile + rm $DIR/$tfile sleep 10 fail ost1 set_nodes_failloc "$(osts_nodes)" 0x0 } run_test 61c "test race mds llog sync vs llog cleanup" +test_61d() { # bug 16002 # bug 17466 + shutdown_facet $SINGLEMDS +#define OBD_FAIL_OBD_LLOG_SETUP 0x605 + do_facet $SINGLEMDS "lctl set_param fail_loc=0x605" + start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS && error "mds start should have failed" + do_facet $SINGLEMDS "lctl set_param fail_loc=0" + start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || error "cannot restart mds" +} +run_test 61d "error in llog_setup should cleanup the llog context correctly" + test_62() { # Bug 15756 - don't mis-drop resent replay mkdir -p $DIR/$tdir replay_barrier $SINGLEMDS createmany -o $DIR/$tdir/$tfile- 25 -#define OBD_FAIL_TGT_REPLAY_DROP 0x706 +#define OBD_FAIL_TGT_REPLAY_DROP 0x707 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000707" facet_failover $SINGLEMDS df $MOUNT || return 1 @@ -1504,10 +1532,31 @@ run_test 62 "don't mis-drop resent replay" #Adaptive Timeouts (bug 3055) AT_MAX_SET=0 -# Suppose that all osts have the same at_max -for facet in mds client ost; do - eval AT_MAX_SAVE_${facet}=$(at_max_get $facet) -done + +at_cleanup () { + local var + local facet + local at_new + + echo "Cleaning up AT ..." + if [ -n "$ATOLDBASE" ]; then + local at_history=$(do_facet mds "find /sys/ -name at_history") + do_facet mds "echo $ATOLDBASE >> $at_history" || true + do_facet ost1 "echo $ATOLDBASE >> $at_history" || true + fi + + if [ $AT_MAX_SET -ne 0 ]; then + for facet in mds client ost; do + var=AT_MAX_SAVE_${facet} + echo restore AT on $facet to saved value ${!var} + at_max_set ${!var} $facet + at_new=$(at_max_get $facet) + echo Restored AT value on $facet $at_new + [ $at_new -eq ${!var} ] || \ + error "$facet : AT value was not restored SAVED ${!var} NEW $at_new" + done + fi +} at_start() { @@ -1517,8 +1566,15 @@ at_start() return 1 fi + # Save at_max original values + local facet + if [ $AT_MAX_SET -eq 0 ]; then + # Suppose that all osts have the same at_max + for facet in mds client ost; do + eval AT_MAX_SAVE_${facet}=$(at_max_get $facet) + done + fi local at_max - for facet in mds client ost; do at_max=$(at_max_get $facet) if [ $at_max -ne $at_max_new ]; then @@ -1544,6 +1600,8 @@ at_start() test_65a() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 $LCTL dk > /dev/null debugsave @@ -1552,7 +1610,7 @@ test_65a() #bug 3055 # because previous tests may have caused this value to increase. REQ_DELAY=`lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | awk '/portal 12/ {print $5}'` - REQ_DELAY=$((${REQ_DELAY} + 5)) + REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5)) do_facet mds lctl set_param fail_val=$((${REQ_DELAY} * 1000)) #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a @@ -1571,6 +1629,8 @@ run_test 65a "AT: verify early replies" test_65b() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 # turn on D_ADAPTTO debugsave @@ -1580,7 +1640,7 @@ test_65b() #bug 3055 # because previous tests may have caused this value to increase. REQ_DELAY=`lctl get_param -n osc.${FSNAME}-OST0000-osc-*.timeouts | awk '/portal 6/ {print $5}'` - REQ_DELAY=$((${REQ_DELAY} + 5)) + REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5)) do_facet ost1 lctl set_param fail_val=${REQ_DELAY} #define OBD_FAIL_OST_BRW_PAUSE_PACK 0x224 @@ -1602,6 +1662,8 @@ run_test 65b "AT: verify early replies on packed reply / bulk" test_66a() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12" # adjust 5s at a time so no early reply is sent (within deadline) @@ -1630,6 +1692,8 @@ run_test 66a "AT: verify MDT service time adjusts with no early replies" test_66b() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 ORIG=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}') sysctl -w lustre.fail_val=$(($ORIG + 5)) @@ -1646,6 +1710,8 @@ run_test 66b "AT: verify net latency adjusts" test_67a() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}') # sleeping threads may drive values above this @@ -1665,6 +1731,8 @@ run_test 67a "AT: verify slow request processing doesn't induce reconnects" test_67b() #bug 3055 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}') #define OBD_FAIL_OST_PAUSE_CREATE 0x223 @@ -1693,44 +1761,38 @@ run_test 67b "AT: verify instant slowdown doesn't induce reconnects" test_68 () #bug 13813 { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + at_start || return 0 local ldlm_enqueue_min=$(find /sys -name ldlm_enqueue_min) [ -z "$ldlm_enqueue_min" ] && skip "missing /sys/.../ldlm_enqueue_min" && return 0 + local ldlm_enqueue_min_r=$(do_facet ost1 "find /sys -name ldlm_enqueue_min") + [ -z "$ldlm_enqueue_min_r" ] && skip "missing /sys/.../ldlm_enqueue_min in the ost1" && return 0 local ENQ_MIN=$(cat $ldlm_enqueue_min) + local ENQ_MIN_R=$(do_facet ost1 "cat $ldlm_enqueue_min_r") echo $TIMEOUT >> $ldlm_enqueue_min - rm -f $DIR/${tfile}_[1-2] - lfs setstripe $DIR/$tfile --index=0 --count=1 + do_facet ost1 "echo $TIMEOUT >> $ldlm_enqueue_min_r" + + rm -rf $DIR/$tdir + mkdir -p $DIR/$tdir + lfs setstripe $DIR/$tdir --index=0 --count=1 #define OBD_FAIL_LDLM_PAUSE_CANCEL 0x312 sysctl -w lustre.fail_val=$(($TIMEOUT - 1)) sysctl -w lustre.fail_loc=0x80000312 - cp /etc/profile $DIR/${tfile}_1 || error "1st cp failed $?" - sysctl -w lustre.fail_val=$((TIMEOUT * 3 / 2)) + cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?" + sysctl -w lustre.fail_val=$((TIMEOUT * 5 / 4)) sysctl -w lustre.fail_loc=0x80000312 - cp /etc/profile $DIR/${tfile}_2 || error "2nd cp failed $?" + cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?" sysctl -w lustre.fail_loc=0 + echo $ENQ_MIN >> $ldlm_enqueue_min + do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r" + rm -rf $DIR/$tdir return 0 } run_test 68 "AT: verify slowing locks" -if [ -n "$ATOLDBASE" ]; then - at_history=$(do_facet mds "find /sys/ -name at_history") - do_facet mds "echo $ATOLDBASE >> $at_history" || true - do_facet ost1 "echo $ATOLDBASE >> $at_history" || true -fi - -if [ $AT_MAX_SET -ne 0 ]; then - for facet in mds client ost; do - var=AT_MAX_SAVE_${facet} - echo restore AT on $facet to saved value ${!var} - at_max_set ${!var} $facet - AT_NEW=$(at_max_get $facet) - echo Restored AT value on $facet $AT_NEW - [ $AT_NEW -ne ${!var} ] && \ - error "$facet : AT value was not restored SAVED ${!var} NEW $AT_NEW" - done -fi - +at_cleanup # end of AT tests includes above lines @@ -1752,7 +1814,7 @@ test_70a () { error "dd failed on $CLIENT" done - local prev_client=$(echo $clients | sed 's/^.* \(.\+\)$/\1/') + local prev_client=$(echo $clients | sed 's/^.* \(.\+\)$/\1/') for C in ${CLIENTS//,/ }; do do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \ error "dd if=$DIR/${tfile}_${C} failed on $prev_client" @@ -1764,42 +1826,101 @@ test_70a () { run_test 70a "check multi client t-f" test_70b () { - [ -z "$CLIENTS" ] && \ - { skip "Need two or more clients." && return; } - [ $CLIENTCOUNT -lt 2 ] && \ - { skip "Need two or more clients, have $CLIENTCOUNT" && return; } + local clients=${CLIENTS:-$HOSTNAME} - zconf_mount_clients $CLIENTS $DIR + zconf_mount_clients $clients $DIR - local duration="-t 60" - local cmd="rundbench 1 $duration " + local duration=300 + [ "$SLOW" = "no" ] && duration=60 + local cmd="rundbench 1 -t $duration" local PID="" - for CLIENT in ${CLIENTS//,/ }; do - $PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" & - PID=$! - echo $PID >pid.$CLIENT - echo "Started load PID=`cat pid.$CLIENT`" + do_nodes $clients "set -x; MISSING_DBENCH_OK=$MISSING_DBENCH_OK \ + PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:$DBENCH_LIB \ + DBENCH_LIB=$DBENCH_LIB TESTSUITE=$TESTSUITE TESTNAME=$TESTNAME \ + LCTL=$LCTL $cmd" & + PID=$! + log "Started rundbench load PID=$PID ..." + ELAPSED=0 + NUM_FAILOVERS=0 + START_TS=$(date +%s) + CURRENT_TS=$START_TS + while [ $ELAPSED -lt $duration ]; do + sleep 1 + replay_barrier $SINGLEMDS + sleep 1 # give clients a time to do operations + # Increment the number of failovers + NUM_FAILOVERS=$((NUM_FAILOVERS+1)) + log "$TESTNAME fail mds1 $NUM_FAILOVERS times" + facet_failover $SINGLEMDS + CURRENT_TS=$(date +%s) + ELAPSED=$((CURRENT_TS - START_TS)) done + wait $PID || error "rundbench load on $CLIENTS failed!" +} +run_test 70b "mds recovery; $CLIENTCOUNT clients" +# end multi-client tests - replay_barrier $SINGLEMDS - sleep 3 # give clients a time to do operations +test_73a() { + multiop_bg_pause $DIR/$tfile O_tSc || return 3 + pid=$! + rm -f $DIR/$tfile - log "$TESTNAME fail mds 1" - fail $SINGLEMDS + replay_barrier $SINGLEMDS +#define OBD_FAIL_LDLM_ENQUEUE 0x302 + do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000302" + fail $SINGLEMDS + kill -USR1 $pid + wait $pid || return 1 + [ -e $DIR/$tfile ] && return 2 + return 0 +} +run_test 73a "open(O_CREAT), unlink, replay, reconnect before open replay , close" -# wait for client to reconnect to MDS - sleep $TIMEOUT +test_73b() { + multiop_bg_pause $DIR/$tfile O_tSc || return 3 + pid=$! + rm -f $DIR/$tfile - for CLIENT in ${CLIENTS//,/ }; do - PID=`cat pid.$CLIENT` - wait $PID - rc=$? - echo "load on ${CLIENT} returned $rc" - done + replay_barrier $SINGLEMDS +#define OBD_FAIL_LDLM_REPLY 0x30c + do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000030c" + fail $SINGLEMDS + kill -USR1 $pid + wait $pid || return 1 + [ -e $DIR/$tfile ] && return 2 + return 0 +} +run_test 73b "open(O_CREAT), unlink, replay, reconnect at open_replay reply, close" + +test_73c() { + multiop_bg_pause $DIR/$tfile O_tSc || return 3 + pid=$! + rm -f $DIR/$tfile + replay_barrier $SINGLEMDS +#define OBD_FAIL_TGT_LAST_REPLAY 0x710 + do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000710" + fail $SINGLEMDS + kill -USR1 $pid + wait $pid || return 1 + [ -e $DIR/$tfile ] && return 2 + return 0 } -run_test 70b "mds recovery; $CLIENTCOUNT clients" -# end multi-client tests +run_test 73c "open(O_CREAT), unlink, replay, reconnect at last_replay, close" + +# bug 18554 +test_74() { + stop ost1 + zconf_umount $(hostname) $MOUNT + fail $SINGLEMDS + zconf_mount $(hostname) $MOUNT + mount_facet ost1 + touch $DIR/$tfile || return 1 + rm $DIR/$tfile || return 2 + df $MOUNT || error "df failed: $?" + return 0 +} +run_test 74 "Ensure applications don't fail waiting for OST reocvery" test_80a() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0 @@ -1809,7 +1930,8 @@ test_80a() { $CHECKSTAT -t dir $DIR/$tdir || error "$CHECKSTAT -t dir $DIR/$tdir failed" rmdir $DIR/$tdir || error "rmdir $DIR/$tdir failed" fail mds2 - stat $DIR/$tdir + stat $DIR/$tdir 2&>/dev/null && error "$DIR/$tdir still exist after recovery!" + return 0 } run_test 80a "CMD: unlink cross-node dir (fail mds with inode)" @@ -1821,7 +1943,8 @@ test_80b() { $CHECKSTAT -t dir $DIR/$tdir || error "$CHECKSTAT -t dir $DIR/$tdir failed" rmdir $DIR/$tdir || error "rmdir $DIR/$tdir failed" fail mds1 - stat $DIR/$tdir + stat $DIR/$tdir 2&>/dev/null && error "$DIR/$tdir still exist after recovery!" + return 0 } run_test 80b "CMD: unlink cross-node dir (fail mds with name)" @@ -1868,4 +1991,4 @@ run_test 82b "CMD: mkdir cross-node dir (fail mds with name)" equals_msg `basename $0`: test complete, cleaning up check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true