+cleanup_26() {
+ trap 0
+ kill -9 $tar_26_pid
+ kill -9 $dbench_26_pid
+ killall -9 dbench
+}
+
+test_26() {
+ local clients=${CLIENTS:-$HOSTNAME}
+
+ zconf_mount_clients $clients $MOUNT
+
+ local duration=600
+ [ "$SLOW" = "no" ] && duration=200
+ # set duration to 900 because it takes some time to boot node
+ [ "$FAILURE_MODE" = HARD ] && duration=900
+
+ local start_ts=$SECONDS
+ local rc=0
+
+ trap cleanup_26 EXIT
+ (
+ local tar_dir=$DIR/$tdir/run_tar
+ while true; do
+ test_mkdir -p -c$MDSCOUNT $tar_dir || break
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS setdirstripe -D -c$MDSCOUNT $tar_dir ||
+ error "set default dirstripe failed"
+ fi
+ cd $tar_dir || break
+ tar cf - /etc | tar xf - || error "tar failed"
+ cd $DIR/$tdir || break
+ rm -rf $tar_dir || break
+ done
+ )&
+ tar_26_pid=$!
+ echo "Started tar $tar_26_pid"
+
+ (
+ local dbench_dir=$DIR2/$tdir/run_dbench
+ while true; do
+ test_mkdir -p -c$MDSCOUNT $dbench_dir || break
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS setdirstripe -D -c$MDSCOUNT $dbench_dir ||
+ error "set default dirstripe failed"
+ fi
+ cd $dbench_dir || break
+ rundbench 1 -D $dbench_dir -t 100 &>/dev/null || break
+ cd $DIR/$tdir || break
+ rm -rf $dbench_dir || break
+ done
+ )&
+ dbench_26_pid=$!
+ echo "Started dbench $dbench_26_pid"
+
+ local num_failovers=0
+ local fail_index=1
+ while [ $((SECONDS - start_ts)) -lt $duration ]; do
+ kill -0 $tar_26_pid || error "tar $tar_26_pid missing"
+ kill -0 $dbench_26_pid || error "dbench $dbench_26_pid missing"
+ sleep 2
+ replay_barrier mds$fail_index
+ sleep 2 # give clients a time to do operations
+ # Increment the number of failovers
+ num_failovers=$((num_failovers + 1))
+ log "$TESTNAME fail mds$fail_index $num_failovers times"
+ fail mds$fail_index
+ if [ $fail_index -ge $MDSCOUNT ]; then
+ fail_index=1
+ else
+ fail_index=$((fail_index + 1))
+ fi
+ done
+ # stop the client loads
+ kill -0 $tar_26_pid || error "tar $tar_26_pid stopped"
+ kill -0 $dbench_26_pid || error "dbench $dbench_26_pid stopped"
+ cleanup_26 || true
+}
+run_test 26 "dbench and tar with mds failover"
+
+test_28() {
+ $LFS setstripe -i 0 -c 1 $DIR2/$tfile
+ dd if=/dev/zero of=$DIR2/$tfile bs=4096 count=1
+
+ #define OBD_FAIL_LDLM_SRV_BL_AST 0x324
+ do_facet ost1 $LCTL set_param fail_loc=0x80000324
+
+ dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1 &
+ local pid=$!
+ sleep 2
+
+ #define OBD_FAIL_LDLM_GRANT_CHECK 0x32a
+ do_facet ost1 $LCTL set_param fail_loc=0x32a
+
+ fail ost1
+
+ sleep 2
+ cancel_lru_locks OST0000-osc
+ wait $pid || error "dd failed"
+}
+run_test 28 "lock replay should be ordered: waiting after granted"
+
+test_29() {
+ local dir0=$DIR/$tdir/d0
+ local dir1=$DIR/$tdir/d1
+
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $CLIENTCOUNT -lt 2 ] && skip "needs >= 2 clients" && return 0
+ [ "$CLIENT1" == "$CLIENT2" ] &&
+ skip "clients must be on different nodes" && return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i0 $dir0
+ $LFS mkdir -i1 $dir1
+ sync
+
+ replay_barrier mds2
+ # create a remote dir, drop reply
+ #define OBD_FAIL_PTLRPC_ROUND_XID 0x530
+ $LCTL set_param fail_loc=0x530 fail_val=36
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet mds2 $LCTL set_param fail_loc=0x8000015a
+ echo make remote dir d0 for $dir0
+ $LFS mkdir -i1 -c1 $dir0/d3 &
+ sleep 1
+
+ echo make local dir d1 for $dir1
+ do_node $CLIENT2 $LCTL set_param fail_loc=0x530 fail_val=36
+ do_node $CLIENT2 mkdir $dir1/d4
+
+ fail mds2
+}
+run_test 29 "replay vs update with the same xid"
+