X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Freplay-single.sh;h=b7c93c882054dab281a64f9c93af4d35e80633a1;hp=b72249ffecfd68ea15fd539a5f4990678e3b9f03;hb=720b42a6c515f301d13e47e1eee38f1c8fc875d9;hpb=e1e5dd6aa49ba49aa2dc58cccf18eff960e1b901

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index b72249f..b7c93c8 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -13,14 +13,15 @@ CLEANUP=${CLEANUP:-}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 CHECK_GRANT=${CHECK_GRANT:-"yes"}
 GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
-remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0
+require_dsh_mds || exit 0
 
 # Skip these tests
-# bug number:  17466
-ALWAYS_EXCEPT="61d   $REPLAY_SINGLE_EXCEPT"
+# bug number:  17466 18857
+ALWAYS_EXCEPT="61d   33a 33b     $REPLAY_SINGLE_EXCEPT"
 
 if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
     CONFIG_EXCEPTIONS="0b 42 47 61a 61c"
@@ -126,7 +127,9 @@ test_0c() {
     touch $DIR/$tfile || return 3
     rm $DIR/$tfile || return 4
 }
+start_full_debug_logging
 run_test 0c "fld create"
+stop_full_debug_logging
 
 test_1() {
     replay_barrier $SINGLEMDS
@@ -157,12 +160,13 @@ test_2b() {
 run_test 2b "touch"
 
 test_3a() {
+    local file=$DIR/$tfile
     replay_barrier $SINGLEMDS
-    mcreate $DIR/$tfile
-    o_directory $DIR/$tfile
+    mcreate $file
+    openfile -f O_DIRECTORY $file
     fail $SINGLEMDS
-    $CHECKSTAT -t file $DIR/$tfile || return 2
-    rm $DIR/$tfile
+    $CHECKSTAT -t file $file || return 2
+    rm $file
 }
 run_test 3a "replay failed open(O_DIRECTORY)"
 
@@ -220,7 +224,7 @@ test_5() {
     done
     fail $SINGLEMDS
     for i in `seq 220`; do
-      grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
+      grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
     done
     rm -rf $DIR/$tfile-*
     sleep 3
@@ -352,6 +356,7 @@ test_13() {
     wait $pid || return 1
 
     $CHECKSTAT -s 1 -p 0 $DIR/$tfile || return 2
+    rm $DIR/$tfile || return 4
     return 0
 }
 run_test 13 "open chmod 0 |x| write close"
@@ -470,15 +475,11 @@ test_20b() { # bug 10480
     lfs getstripe $DIR/$tfile || return 1
     rm -f $DIR/$tfile || return 2       # make it an orphan
     mds_evict_client
-    df -P $DIR || df -P $DIR || true    # reconnect
+    client_up || client_up || true    # reconnect
 
     fail $SINGLEMDS                            # start orphan recovery
-    df -P $DIR || df -P $DIR || true    # reconnect
-    wait_mds_recovery_done || error "MDS recovery not done"
-
-    # FIXME just because recovery is done doesn't mean we've finished
-    # orphan cleanup.  Fake it with a sleep for now...
-    sleep 10
+    wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
+    wait_mds_ost_sync || return 3
     AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
     log "before $BEFOREUSED, after $AFTERUSED"
     [ $AFTERUSED -gt $((BEFOREUSED + 20)) ] && \
@@ -494,13 +495,12 @@ test_20c() { # bug 10480
     ls -la $DIR/$tfile
 
     mds_evict_client
-
-    df -P $DIR || df -P $DIR || true    # reconnect
+    client_up || client_up || true    # reconnect
 
     kill -USR1 $pid
-    test -s $DIR/$tfile || error "File was truncated"
-
     wait $pid || return 1
+    [ -s $DIR/$tfile ] || error "File was truncated"
+
     return 0
 }
 run_test 20c "check that client eviction does not affect file content"
@@ -709,7 +709,7 @@ test_32() {
     multiop_bg_pause $DIR/$tfile O_c || return 3
     pid2=$!
     mds_evict_client
-    df $MOUNT || sleep 1 && df $MOUNT || return 1
+    client_up || client_up || return 1
     kill -USR1 $pid1
     kill -USR1 $pid2
     wait $pid1 || return 4
@@ -718,30 +718,30 @@ test_32() {
 }
 run_test 32 "close() notices client eviction; close() after client eviction"
 
-# Abort recovery before client complete
-test_33a() {	# was test_33
-    replay_barrier $SINGLEMDS
-    createmany -o $DIR/$tfile-%d 100
+test_33a() {
+    createmany -o $DIR/$tfile-%d 10
+    replay_barrier_nosync $SINGLEMDS
     fail_abort $SINGLEMDS
-    # this file should be gone, because the replay was aborted
-    $CHECKSTAT -t file $DIR/$tfile-* && return 3
-    unlinkmany $DIR/$tfile-%d 0 100
+    # recreate shouldn't fail
+    createmany -o $DIR/$tfile--%d 10 || return 1
+    rm $DIR/$tfile-* -f
     return 0
 }
-run_test 33a "abort recovery before client does replay"
+run_test 33a "fid seq shouldn't be reused after abort recovery"
+
+test_33b() {
+    #define OBD_FAIL_SEQ_ALLOC                          0x1311
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x1311"
 
-# Stale FID sequence
-test_33b() {	# was test_33a
-    replay_barrier $SINGLEMDS
     createmany -o $DIR/$tfile-%d 10
+    replay_barrier_nosync $SINGLEMDS
     fail_abort $SINGLEMDS
-    unlinkmany $DIR/$tfile-%d 0 10
     # recreate shouldn't fail
-    createmany -o $DIR/$tfile-%d 10 || return 3
-    unlinkmany $DIR/$tfile-%d 0 10
+    createmany -o $DIR/$tfile--%d 10 || return 1
+    rm $DIR/$tfile-* -f
     return 0
 }
-run_test 33b "fid shouldn't be reused after abort recovery"
+run_test 33b "test fid seq allocation"
 
 test_34() {
     multiop_bg_pause $DIR/$tfile O_c || return 2
@@ -807,7 +807,9 @@ test_37() {
     sync
     return 0
 }
+start_full_debug_logging
 run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans for directories)"
+stop_full_debug_logging
 
 test_38() {
     createmany -o $DIR/$tfile-%d 800
@@ -880,7 +882,7 @@ run_test 40 "cause recovery in ptlrpc, ensure IO continues"
 # assert on trying to unlock the unlocked page.
 test_41() {
     [ $OSTCOUNT -lt 2 ] && \
-	skip "skipping test 41: we don't have a second OST to test with" && \
+	skip_env "skipping test 41: we don't have a second OST to test with" && \
 	return
 
     local f=$MOUNT/$tfile
@@ -935,25 +937,27 @@ test_43() { # bug 2530
 }
 run_test 43 "mds osc import failure during recovery; don't LBUG"
 
-test_44a() {	# was test_44
+test_44a() { # was test_44
     local at_max_saved=0
 
     mdcdev=`lctl get_param -n devices | awk '/MDT0000-mdc-/ {print $1}'`
-    [ "$mdcdev" ] || exit 2
+    [ "$mdcdev" ] || return 2
+    [ $(echo $mdcdev | wc -w) -eq 1 ] || { echo $mdcdev=$mdcdev && return 3; }
 
     # adaptive timeouts slow this way down
-    if at_is_valid && at_is_enabled; then
+    if at_is_enabled; then
         at_max_saved=$(at_max_get mds)
         at_max_set 40 mds
     fi
 
     for i in `seq 1 10`; do
-	echo "$i of 10 ($(date +%s))"
-	do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
-	#define OBD_FAIL_TGT_CONN_RACE     0x701
-	do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000701"
-	$LCTL --device $mdcdev recover
-	df $MOUNT
+        echo "$i of 10 ($(date +%s))"
+        do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
+        #define OBD_FAIL_TGT_CONN_RACE     0x701
+        do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000701"
+        # lctl below may fail, it is valid case
+        $LCTL --device $mdcdev recover
+        df $MOUNT
     done
     do_facet $SINGLEMDS "lctl set_param fail_loc=0"
     [ $at_max_saved -ne 0 ] && at_max_set $at_max_saved mds
@@ -962,15 +966,18 @@ test_44a() {	# was test_44
 run_test 44a "race in target handle connect"
 
 test_44b() {
-    mdcdev=`lctl get_param -n devices | awk '/MDT0000-mdc-/ {print $1}'`
-    [ "$mdcdev" ] || exit 2
+    local mdcdev=`lctl get_param -n devices | awk '/MDT0000-mdc-/ {print $1}'`
+    [ "$mdcdev" ] || return 2
+    [ $(echo $mdcdev | wc -w) -eq 1 ] || { echo $mdcdev=$mdcdev && return 3; }
+
     for i in `seq 1 10`; do
         echo "$i of 10 ($(date +%s))"
-	do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
-	#define OBD_FAIL_TGT_DELAY_RECONNECT 0x704
-	do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000704"
-	$LCTL --device $mdcdev recover
-	df $MOUNT
+        do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
+        #define OBD_FAIL_TGT_DELAY_RECONNECT 0x704
+        do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000704"
+        # lctl below may fail, it is valid case
+        $LCTL --device $mdcdev recover
+        df $MOUNT
     done
     do_facet $SINGLEMDS "lctl set_param fail_loc=0"
     return 0
@@ -980,8 +987,10 @@ run_test 44b "race in target handle connect"
 # Handle failed close
 test_45() {
     mdcdev=`lctl get_param -n devices | awk '/MDT0000-mdc-/ {print $1}'`
-    [ "$mdcdev" ] || exit 2
-    $LCTL --device $mdcdev recover
+    [ "$mdcdev" ] || return 2
+    [ $(echo $mdcdev | wc -w) -eq 1 ] || { echo $mdcdev=$mdcdev && return 3; }
+
+    $LCTL --device $mdcdev recover || return 6 
 
     multiop_bg_pause $DIR/$tfile O_c || return 1
     pid=$!
@@ -1023,7 +1032,7 @@ test_47() { # bug 2824
     # OBD_FAIL_OST_CREATE_NET 0x204
     fail ost1
     do_facet ost1 "lctl set_param fail_loc=0x80000204"
-    df $MOUNT || return 2
+    client_up || return 2
 
     # let the MDS discover the OST failure, attempt to recover, fail
     # and recover again.
@@ -1040,14 +1049,14 @@ run_test 47 "MDS->OSC failure during precreate cleanup (2824)"
 
 test_48() {
     remote_ost_nodsh && skip "remote OST with nodsh" && return 0
-    [ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2 OSTs -- skipping" && return
+    [ "$OSTCOUNT" -lt "2" ] && skip_env "$OSTCOUNT < 2 OSTs -- skipping" && return
 
     replay_barrier $SINGLEMDS
     createmany -o $DIR/$tfile 20  || return 1
     # OBD_FAIL_OST_EROFS 0x216
     facet_failover $SINGLEMDS
     do_facet ost1 "lctl set_param fail_loc=0x80000216"
-    df $MOUNT || return 2
+    client_up || return 2
 
     createmany -o $DIR/$tfile 20 20 || return 2
     unlinkmany $DIR/$tfile 40 || return 3
@@ -1083,6 +1092,7 @@ run_test 52 "time out lock replay (3764)"
 
 # bug 3462 - simultaneous MDC requests
 test_53a() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1112,9 +1122,12 @@ test_53a() {
 run_test 53a "|X| close request while two MDC requests in flight"
 
 test_53b() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
-        multiop $DIR/${tdir}-1/f O_c &
+        multiop_bg_pause $DIR/${tdir}-1/f O_c || return 6
         close_pid=$!
 
         #define OBD_FAIL_MDS_REINT_NET 0x107
@@ -1141,6 +1154,9 @@ test_53b() {
 run_test 53b "|X| open request while two MDC requests in flight"
 
 test_53c() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1157,6 +1173,9 @@ test_53c() {
         kill -USR1 $close_pid
         cancel_lru_locks mdc    # force the close
 
+        #bz20647: make sure all pids are exists before failover
+        [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
+        [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
         replay_barrier_nodf $SINGLEMDS
         fail_nodf $SINGLEMDS
         wait $open_pid || return 1
@@ -1172,6 +1191,9 @@ test_53c() {
 run_test 53c "|X| open request and close request while two MDC requests in flight"
 
 test_53d() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1179,8 +1201,8 @@ test_53d() {
         # give multiop a chance to open
         sleep 1
 
-        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f
-        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f"
+        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
+        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
         kill -USR1 $close_pid
         cancel_lru_locks mdc    # force the close
         do_facet $SINGLEMDS "lctl set_param fail_loc=0"
@@ -1198,6 +1220,9 @@ test_53d() {
 run_test 53d "|X| close reply while two MDC requests in flight"
 
 test_53e() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1227,6 +1252,9 @@ test_53e() {
 run_test 53e "|X| open reply while two MDC requests in flight"
 
 test_53f() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1238,11 +1266,14 @@ test_53f() {
         open_pid=$!
         sleep 1
 
-        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f
-        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f"
+        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
+        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
         kill -USR1 $close_pid
         cancel_lru_locks mdc    # force the close
 
+        #bz20647: make sure all pids are exists before failover
+        [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
+        [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
         replay_barrier_nodf $SINGLEMDS
         fail_nodf $SINGLEMDS
         wait $open_pid || return 1
@@ -1258,6 +1289,9 @@ test_53f() {
 run_test 53f "|X| open reply and close reply while two MDC requests in flight"
 
 test_53g() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1273,8 +1307,11 @@ test_53g() {
         do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
         kill -USR1 $close_pid
         cancel_lru_locks mdc    # force the close
-
         do_facet $SINGLEMDS "lctl set_param fail_loc=0"
+
+        #bz20647: make sure all pids are exists before failover
+        [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
+        [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
         replay_barrier_nodf $SINGLEMDS
         fail_nodf $SINGLEMDS
         wait $open_pid || return 1
@@ -1289,6 +1326,9 @@ test_53g() {
 run_test 53g "|X| drop open reply and close request while close and open are both in flight"
 
 test_53h() {
+        cancel_lru_locks mdc    # cleanup locks from former test cases
+        rm -rf $DIR/${tdir}-1 $DIR/${tdir}-2
+
         mkdir -p $DIR/${tdir}-1
         mkdir -p $DIR/${tdir}-2
         multiop $DIR/${tdir}-1/f O_c &
@@ -1300,12 +1340,15 @@ test_53h() {
         open_pid=$!
         sleep 1
 
-        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13f
-        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013f"
+        #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
+        do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
         kill -USR1 $close_pid
         cancel_lru_locks mdc    # force the close
         sleep 1
 
+        #bz20647: make sure all pids are exists before failover
+        [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
+        [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
         replay_barrier_nodf $SINGLEMDS
         fail_nodf $SINGLEMDS
         wait $open_pid || return 1
@@ -1427,33 +1470,6 @@ test_59() {
 }
 run_test 59 "test log_commit_thread vs filter_destroy race"
 
-# bug 17323
-test_59b() {
-    do_facet $SINGLEMDS "lctl set_param debug=+rpctrace"
-    mkdir -p $DIR/$tdir
-    createmany -o $DIR/$tdir/$tfile-%d 2000
-    sync
-#define OBD_FAIL_OBD_LOG_CANCEL_REP      0x606
-    do_facet $SINGLEMDS "lctl set_param fail_loc=0x606"
-    unlinkmany $DIR/$tdir/$tfile-%d 2000
-
-    # make sure that all llcds left ost and nothing left cached
-    sync
-    sleep 10
-    do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
-
-    # sleep 2 obd_timeouts from ost to make sure that we get resents.
-    local timeout=$(do_facet ost1 lctl get_param -n timeout)
-    timeout=$((timeout * 2))
-    log "Sleep $timeout"
-    sleep $timeout
-    do_facet $SINGLEMDS $LCTL dk | grep -q "RESENT cancel req"
-    local res=$?
-    rmdir $DIR/$tdir
-    return $res
-}
-run_test 59b "resent handle in llog_origin_handle_cancel"
-
 # race between add unlink llog vs cat log init in post_recovery (only for b1_6)
 # bug 12086: should no oops and No ctxt error for this test
 test_60() {
@@ -1472,18 +1488,18 @@ run_test 60 "test llog post recovery init vs llog unlink"
 test_61a() {	# was test_61
     remote_ost_nodsh && skip "remote OST with nodsh" && return 0
 
-    mkdir $DIR/$tdir
+    mkdir -p $DIR/$tdir
     createmany -o $DIR/$tdir/$tfile-%d 800
-    replay_barrier ost1 
-#   OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221 
-    unlinkmany $DIR/$tdir/$tfile-%d 800 
+    replay_barrier ost1
+#   OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221
+    unlinkmany $DIR/$tdir/$tfile-%d 800
     set_nodes_failloc "$(osts_nodes)" 0x80000221
     facet_failover ost1
-    sleep 10 
+    sleep 10
     fail ost1
     sleep 30
     set_nodes_failloc "$(osts_nodes)" 0x0
-    
+
     $CHECKSTAT -t file $DIR/$tdir/$tfile-* && return 1
     rmdir $DIR/$tdir
 }
@@ -1491,9 +1507,9 @@ run_test 61a "test race llog recovery vs llog cleanup"
 
 #test race  mds llog sync vs llog cleanup
 test_61b() {
-#   OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x140
-    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000140"
-    facet_failover $SINGLEMDS 
+#   OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x13a
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013a"
+    facet_failover $SINGLEMDS
     sleep 10
     fail $SINGLEMDS
     do_facet client dd if=/dev/zero of=$DIR/$tfile bs=4k count=1 || return 1
@@ -1504,24 +1520,33 @@ run_test 61b "test race mds llog sync vs llog cleanup"
 test_61c() {
     remote_ost_nodsh && skip "remote OST with nodsh" && return 0
 
-#   OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222 
-    touch $DIR/$tfile 
+#   OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222
+    touch $DIR/$tfile
     set_nodes_failloc "$(osts_nodes)" 0x80000222
-    rm $DIR/$tfile    
+    rm $DIR/$tfile
     sleep 10
     fail ost1
     set_nodes_failloc "$(osts_nodes)" 0x0
 }
 run_test 61c "test race mds llog sync vs llog cleanup"
 
+test_61d() { # bug 16002 # bug 17466
+    shutdown_facet $SINGLEMDS
+#define OBD_FAIL_OBD_LLOG_SETUP        0x605
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x605"
+    start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS && error "mds start should have failed"
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0"
+    start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || error "cannot restart mds"
+}
+run_test 61d "error in llog_setup should cleanup the llog context correctly"
+
 test_62() { # Bug 15756 - don't mis-drop resent replay
     mkdir -p $DIR/$tdir
     replay_barrier $SINGLEMDS
     createmany -o $DIR/$tdir/$tfile- 25
-#define OBD_FAIL_TGT_REPLAY_DROP         0x706
+#define OBD_FAIL_TGT_REPLAY_DROP         0x707
     do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000707"
-    facet_failover $SINGLEMDS
-    df $MOUNT || return 1
+    fail $SINGLEMDS
     do_facet $SINGLEMDS "lctl set_param fail_loc=0"
     unlinkmany $DIR/$tdir/$tfile- 25 || return 2
     return 0
@@ -1530,21 +1555,45 @@ run_test 62 "don't mis-drop resent replay"
 
 #Adaptive Timeouts (bug 3055)
 AT_MAX_SET=0
-# Suppose that all osts have the same at_max
-for facet in mds client ost; do
-    eval AT_MAX_SAVE_${facet}=$(at_max_get $facet)
-done
+
+at_cleanup () {
+    local var
+    local facet
+    local at_new
+
+    echo "Cleaning up AT ..."
+    if [ -n "$ATOLDBASE" ]; then
+        local at_history=$($LCTL get_param -n at_history)
+        do_facet mds "lctl set_param at_history=$at_history" || true
+        do_facet ost1 "lctl set_param at_history=$at_history" || true
+    fi
+
+    if [ $AT_MAX_SET -ne 0 ]; then
+        for facet in mds client ost; do
+            var=AT_MAX_SAVE_${facet}
+            echo restore AT on $facet to saved value ${!var}
+            at_max_set ${!var} $facet
+            at_new=$(at_max_get $facet)
+            echo Restored AT value on $facet $at_new
+            [ $at_new -eq ${!var} ] || \
+            error "$facet : AT value was not restored SAVED ${!var} NEW $at_new"
+        done
+    fi
+}
 
 at_start()
 {
     local at_max_new=600
-    if ! at_is_valid; then
-        skip "AT env is invalid"
-        return 1
-    fi
 
+    # Save at_max original values
+    local facet
+    if [ $AT_MAX_SET -eq 0 ]; then
+        # Suppose that all osts have the same at_max
+        for facet in mds client ost; do
+            eval AT_MAX_SAVE_${facet}=$(at_max_get $facet)
+        done
+    fi
     local at_max
-
     for facet in mds client ost; do
         at_max=$(at_max_get $facet)
         if [ $at_max -ne $at_max_new ]; then
@@ -1555,12 +1604,10 @@ at_start()
     done
 
     if [ -z "$ATOLDBASE" ]; then
-	local at_history=$(do_facet mds "find /sys/ -name at_history")
-	[ -z "$at_history" ] && skip "missing /sys/.../at_history " && return 1
-	ATOLDBASE=$(do_facet mds "cat $at_history")
+	ATOLDBASE=$(do_facet mds "lctl get_param -n at_history")
         # speed up the timebase so we can check decreasing AT
-	do_facet mds "echo 8 >> $at_history"
-	do_facet ost1 "echo 8 >> $at_history"
+        do_facet mds "lctl set_param at_history=8" || true
+        do_facet ost1 "lctl set_param at_history=8" || true
 
 	# sleep for a while to cool down, should be > 8s and also allow
 	# at least one ping to be sent. simply use TIMEOUT to be safe.
@@ -1580,7 +1627,7 @@ test_65a() #bug 3055
     # because previous tests may have caused this value to increase.
     REQ_DELAY=`lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts |
                awk '/portal 12/ {print $5}'`
-    REQ_DELAY=$((${REQ_DELAY} + 5))
+    REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5))
 
     do_facet mds lctl set_param fail_val=$((${REQ_DELAY} * 1000))
 #define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
@@ -1608,9 +1655,11 @@ test_65b() #bug 3055
     $LCTL dk > /dev/null
     # Slow down a request to the current service time, this is critical
     # because previous tests may have caused this value to increase.
+    lfs setstripe $DIR/$tfile --index=0 --count=1
+    multiop $DIR/$tfile Ow1yc
     REQ_DELAY=`lctl get_param -n osc.${FSNAME}-OST0000-osc-*.timeouts |
                awk '/portal 6/ {print $5}'`
-    REQ_DELAY=$((${REQ_DELAY} + 5))
+    REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5))
 
     do_facet ost1 lctl set_param fail_val=${REQ_DELAY}
 #define OBD_FAIL_OST_BRW_PAUSE_PACK      0x224
@@ -1693,7 +1742,7 @@ test_67a() #bug 3055
     do_facet ost1 "sysctl -w lustre.fail_loc=0"
     CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     ATTEMPTS=$(($CONN2 - $CONN1))
-    echo "$ATTEMPTS osc reconnect attemps on gradual slow"
+    echo "$ATTEMPTS osc reconnect attempts on gradual slow"
     [ $ATTEMPTS -gt 0 ] && error_ignore 13721 "AT should have prevented reconnect"
     return 0
 }
@@ -1705,16 +1754,27 @@ test_67b() #bug 3055
 
     at_start || return 0
     CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
+
+    # exhaust precreations on ost1
+    local OST=$(lfs osts | grep ^0": " | awk '{print $2}' | sed -e 's/_UUID$//')
+    local mdtosc=$(get_mdtosc_proc_path $OST)
+    local last_id=$(do_facet mds lctl get_param -n osc.$mdtosc.prealloc_last_id)
+    local next_id=$(do_facet mds lctl get_param -n osc.$mdtosc.prealloc_next_id)
+
+    mkdir -p $DIR/$tdir/${OST}
+    lfs setstripe $DIR/$tdir/${OST} -o 0 -c 1 || error "setstripe"
+    echo "Creating to objid $last_id on ost $OST..."
 #define OBD_FAIL_OST_PAUSE_CREATE        0x223
     do_facet ost1 "sysctl -w lustre.fail_val=20000"
     do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
-    cp /etc/profile $DIR/$tfile || error "cp failed"
+    createmany -o $DIR/$tdir/${OST}/f $next_id $((last_id - next_id + 2))
+
     client_reconnect
     do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
     log "phase 2"
     CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     ATTEMPTS=$(($CONN2 - $CONN1))
-    echo "$ATTEMPTS osc reconnect attemps on instant slow"
+    echo "$ATTEMPTS osc reconnect attempts on instant slow"
     # do it again; should not timeout
     do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
     cp /etc/profile $DIR/$tfile || error "cp failed"
@@ -1723,7 +1783,7 @@ test_67b() #bug 3055
     do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
     CONN3=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     ATTEMPTS=$(($CONN3 - $CONN2))
-    echo "$ATTEMPTS osc reconnect attemps on 2nd slow"
+    echo "$ATTEMPTS osc reconnect attempts on 2nd slow"
     [ $ATTEMPTS -gt 0 ] && error "AT should have prevented reconnect"
     return 0
 }
@@ -1736,41 +1796,33 @@ test_68 () #bug 13813
     at_start || return 0
     local ldlm_enqueue_min=$(find /sys -name ldlm_enqueue_min)
     [ -z "$ldlm_enqueue_min" ] && skip "missing /sys/.../ldlm_enqueue_min" && return 0
+    local ldlm_enqueue_min_r=$(do_facet ost1 "find /sys -name ldlm_enqueue_min")
+    [ -z "$ldlm_enqueue_min_r" ] && skip "missing /sys/.../ldlm_enqueue_min in the ost1" && return 0
     local ENQ_MIN=$(cat $ldlm_enqueue_min)
+    local ENQ_MIN_R=$(do_facet ost1 "cat $ldlm_enqueue_min_r")
     echo $TIMEOUT >> $ldlm_enqueue_min
-    rm -f $DIR/${tfile}_[1-2]
-    lfs setstripe $DIR/$tfile --index=0 --count=1
+    do_facet ost1 "echo $TIMEOUT >> $ldlm_enqueue_min_r"
+
+    rm -rf $DIR/$tdir
+    mkdir -p $DIR/$tdir
+    lfs setstripe $DIR/$tdir --index=0 --count=1
 #define OBD_FAIL_LDLM_PAUSE_CANCEL       0x312
     sysctl -w lustre.fail_val=$(($TIMEOUT - 1))
     sysctl -w lustre.fail_loc=0x80000312
-    cp /etc/profile $DIR/${tfile}_1 || error "1st cp failed $?"
-    sysctl -w lustre.fail_val=$((TIMEOUT * 3 / 2))
+    cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?"
+    sysctl -w lustre.fail_val=$((TIMEOUT * 5 / 4))
     sysctl -w lustre.fail_loc=0x80000312
-    cp /etc/profile $DIR/${tfile}_2 || error "2nd cp failed $?"
+    cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?"
     sysctl -w lustre.fail_loc=0
+
     echo $ENQ_MIN >> $ldlm_enqueue_min
+    do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r"
+    rm -rf $DIR/$tdir
     return 0
 }
 run_test 68 "AT: verify slowing locks"
 
-if [ -n "$ATOLDBASE" ]; then
-    at_history=$(do_facet mds "find /sys/ -name at_history")
-    do_facet mds "echo $ATOLDBASE >> $at_history" || true
-    do_facet ost1 "echo $ATOLDBASE >> $at_history" || true
-fi
-
-if [ $AT_MAX_SET -ne 0 ]; then
-    for facet in mds client ost; do
-        var=AT_MAX_SAVE_${facet}
-        echo restore AT on $facet to saved value ${!var}
-        at_max_set ${!var} $facet
-        AT_NEW=$(at_max_get $facet)
-        echo Restored AT value on $facet $AT_NEW 
-        [ $AT_NEW -ne ${!var} ] && \
-            error "$facet : AT value was not restored SAVED ${!var} NEW $AT_NEW"
-    done
-fi
-
+at_cleanup
 # end of AT tests includes above lines
 
 
@@ -1792,7 +1844,7 @@ test_70a () {
 				error "dd failed on $CLIENT"
 	done
 
-	local prev_client=$(echo $clients | sed 's/^.* \(.\+\)$/\1/') 
+	local prev_client=$(echo $clients | sed 's/^.* \(.\+\)$/\1/')
 	for C in ${CLIENTS//,/ }; do
 		do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \
 			error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
@@ -1804,42 +1856,103 @@ test_70a () {
 run_test 70a "check multi client t-f"
 
 test_70b () {
-	[ -z "$CLIENTS" ] && \
-		{ skip "Need two or more clients." && return; }
-	[ $CLIENTCOUNT -lt 2 ] && \
-		{ skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+	local clients=${CLIENTS:-$HOSTNAME}
 
-	zconf_mount_clients $CLIENTS $DIR
+	zconf_mount_clients $clients $DIR
 	
-	local duration="-t 60"
-	local cmd="rundbench 1 $duration "
+	local duration=300
+	[ "$SLOW" = "no" ] && duration=60
+	local cmd="rundbench 1 -t $duration"
 	local PID=""
-	for CLIENT in ${CLIENTS//,/ }; do
-		$PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
-		PID=$!
-		echo $PID >pid.$CLIENT
-		echo "Started load PID=`cat pid.$CLIENT`"
+	do_nodesv $clients "set -x; MISSING_DBENCH_OK=$MISSING_DBENCH_OK \
+		PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:$DBENCH_LIB \
+		DBENCH_LIB=$DBENCH_LIB TESTSUITE=$TESTSUITE TESTNAME=$TESTNAME \
+		LCTL=$LCTL $cmd" &
+	PID=$!
+	log "Started rundbench load PID=$PID ..."
+	ELAPSED=0
+	NUM_FAILOVERS=0
+	START_TS=$(date +%s)
+	CURRENT_TS=$START_TS
+	while [ $ELAPSED -lt $duration ]; do
+		sleep 1
+		replay_barrier $SINGLEMDS
+		sleep 1 # give clients a time to do operations
+		# Increment the number of failovers
+		NUM_FAILOVERS=$((NUM_FAILOVERS+1))
+		log "$TESTNAME fail mds1 $NUM_FAILOVERS times"
+		fail $SINGLEMDS
+		CURRENT_TS=$(date +%s)
+		ELAPSED=$((CURRENT_TS - START_TS))
 	done
+	wait $PID || error "rundbench load on $CLIENTS failed!"
+}
+run_test 70b "mds recovery; $CLIENTCOUNT clients"
+# end multi-client tests
 
-	replay_barrier $SINGLEMDS 
-	sleep 3 # give clients a time to do operations
+test_73a() {
+    multiop_bg_pause $DIR/$tfile O_tSc || return 3
+    pid=$!
+    rm -f $DIR/$tfile
 
-	log "$TESTNAME fail mds 1"
-	fail $SINGLEMDS
+    replay_barrier $SINGLEMDS
+#define OBD_FAIL_LDLM_ENQUEUE       0x302
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000302"
+    fail $SINGLEMDS
+    kill -USR1 $pid
+    wait $pid || return 1
+    [ -e $DIR/$tfile ] && return 2
+    return 0
+}
+run_test 73a "open(O_CREAT), unlink, replay, reconnect before open replay , close"
 
-# wait for client to reconnect to MDS
-	sleep $TIMEOUT
+test_73b() {
+    multiop_bg_pause $DIR/$tfile O_tSc || return 3
+    pid=$!
+    rm -f $DIR/$tfile
 
-	for CLIENT in ${CLIENTS//,/ }; do
-		PID=`cat pid.$CLIENT`
-		wait $PID
-		rc=$?
-		echo "load on ${CLIENT} returned $rc"
-	done
+    replay_barrier $SINGLEMDS
+#define OBD_FAIL_LDLM_REPLY       0x30c
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000030c"
+    fail $SINGLEMDS
+    kill -USR1 $pid
+    wait $pid || return 1
+    [ -e $DIR/$tfile ] && return 2
+    return 0
+}
+run_test 73b "open(O_CREAT), unlink, replay, reconnect at open_replay reply, close"
+
+test_73c() {
+    multiop_bg_pause $DIR/$tfile O_tSc || return 3
+    pid=$!
+    rm -f $DIR/$tfile
 
+    replay_barrier $SINGLEMDS
+#define OBD_FAIL_TGT_LAST_REPLAY       0x710
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000710"
+    fail $SINGLEMDS
+    kill -USR1 $pid
+    wait $pid || return 1
+    [ -e $DIR/$tfile ] && return 2
+    return 0
 }
-run_test 70b "mds recovery; $CLIENTCOUNT clients"
-# end multi-client tests
+run_test 73c "open(O_CREAT), unlink, replay, reconnect at last_replay, close"
+
+# bug 18554
+test_74() {
+    local clients=${CLIENTS:-$HOSTNAME}
+
+    stop ost1
+    zconf_umount_clients $clients $MOUNT
+    facet_failover $SINGLEMDS
+    zconf_mount_clients $clients $MOUNT
+    mount_facet ost1
+    touch $DIR/$tfile || return 1
+    rm $DIR/$tfile || return 2
+    clients_up || error "client evicted: $?"
+    return 0
+}
+run_test 74 "Ensure applications don't fail waiting for OST recovery"
 
 test_80a() {
     [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
@@ -1849,7 +1962,8 @@ test_80a() {
     $CHECKSTAT -t dir $DIR/$tdir || error "$CHECKSTAT -t dir $DIR/$tdir failed"
     rmdir $DIR/$tdir || error "rmdir $DIR/$tdir failed"
     fail mds2
-    stat $DIR/$tdir
+    stat $DIR/$tdir 2&>/dev/null && error "$DIR/$tdir still exist after recovery!"
+    return 0
 }
 run_test 80a "CMD: unlink cross-node dir (fail mds with inode)"
 
@@ -1861,7 +1975,8 @@ test_80b() {
     $CHECKSTAT -t dir $DIR/$tdir || error "$CHECKSTAT -t dir $DIR/$tdir failed"
     rmdir $DIR/$tdir || error "rmdir $DIR/$tdir failed"
     fail mds1
-    stat $DIR/$tdir
+    stat $DIR/$tdir 2&>/dev/null && error "$DIR/$tdir still exist after recovery!"
+    return 0
 }
 run_test 80b "CMD: unlink cross-node dir (fail mds with name)"
 
@@ -1906,6 +2021,57 @@ test_82b() {
 }
 run_test 82b "CMD: mkdir cross-node dir (fail mds with name)"
 
+test_83a() {
+    mkdir -p $DIR/$tdir
+    createmany -o $DIR/$tdir/$tfile- 10 || return 1
+#define OBD_FAIL_MDS_FAIL_LOV_LOG_ADD       0x140
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000140"
+    unlinkmany $DIR/$tdir/$tfile- 10 || return 2
+}
+run_test 83a "fail log_add during unlink recovery"
+
+test_83b() {
+    mkdir -p $DIR/$tdir
+    createmany -o $DIR/$tdir/$tfile- 10 || return 1
+    replay_barrier $SINGLEMDS
+    unlinkmany $DIR/$tdir/$tfile- 10 || return 2
+#define OBD_FAIL_MDS_FAIL_LOV_LOG_ADD       0x140
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000140"
+    fail $SINGLEMDS
+}
+run_test 83b "fail log_add during unlink recovery"
+
+test_84a() {
+#define OBD_FAIL_MDS_OPEN_WAIT_CREATE  0x144
+    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000144"
+    createmany -o $DIR/$tfile- 1 &
+    PID=$!
+    mds_evict_client
+    wait $PID
+    client_up || client_up || true    # reconnect
+}
+run_test 84a "stale open during export disconnect"
+
+test_85() { # bug 22190
+    local fail=0
+    do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.sync_journal 1"
+
+    replay_barrier ost1
+    lfs setstripe -i 0 -c 1 $DIR/$tfile
+    dd oflag=dsync if=/dev/urandom of=$DIR/$tfile bs=4k count=100 || fail=1
+    fail_abort ost1
+    echo "FAIL $fail"
+    [ $fail -ne 0 ] || error "Write was successful"
+}
+run_test 85 "ensure there is no reply on bulk write if obd is in rdonly mode"
+
+test_86() {
+        umount $MOUNT
+        do_facet $SINGLEMDS lctl set_param mdt.${FSNAME}-MDT*.exports.clear=0
+        remount_facet $SINGLEMDS
+}
+run_test 86 "umount server after clear nid_stats should not hit LBUG"
+
 equals_msg `basename $0`: test complete, cleaning up
 check_and_cleanup_lustre
-[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true