X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=inline;f=lustre%2Ftests%2Freplay-single.sh;h=9998498710fc1b58695cf114b4674d4f0b0126f7;hb=1bf48948a99f51ede84a467a23c59c55eb055d7b;hp=d697853db8e3e30184b2562ad539b78840be1b43;hpb=2f93aa4e73148b2d8daeea974a82effc5fcf0321;p=fs%2Flustre-release.git

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index d697853..9998498 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -21,8 +21,10 @@ GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 require_dsh_mds || exit 0
 
 # Skip these tests
-# bug number for skipped tests: b=17466/LU-472
-ALWAYS_EXCEPT="                 61d	$REPLAY_SINGLE_EXCEPT"
+# bug number for skipped tests:
+# b=17466/LU-472 : 61d
+# LU-5319 : 53a 53d
+ALWAYS_EXCEPT="61d 53a 53d $REPLAY_SINGLE_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 case "$(lsb_release -sr)" in	# only disable tests for el7
@@ -2025,15 +2027,6 @@ check_for_process () {
 	killall_process $clients "$prog" -0
 }
 
-killall_process () {
-	local clients=${1:-$(hostname)}
-	local name=$2
-	local signal=$3
-	local rc=0
-
-	do_nodes $clients "killall $signal $name"
-}
-
 test_70b () {
 	local clients=${CLIENTS:-$HOSTNAME}
 
@@ -2048,6 +2041,10 @@ test_70b () {
 	local start_ts=$(date +%s)
 	local cmd="rundbench 1 -t $duration"
 	local pid=""
+	if [ $MDSCOUNT -ge 2 ]; then
+		test_mkdir -p -c$MDSCOUNT $DIR/$tdir
+		$LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir
+	fi
 	do_nodesv $clients "set -x; MISSING_DBENCH_OK=$MISSING_DBENCH_OK \
 		PATH=\$PATH:$LUSTRE/utils:$LUSTRE/tests/:$DBENCH_LIB \
 		DBENCH_LIB=$DBENCH_LIB TESTSUITE=$TESTSUITE TESTNAME=$TESTNAME \
@@ -2068,6 +2065,7 @@ test_70b () {
 
 	elapsed=$(($(date +%s) - start_ts))
 	local num_failovers=0
+	local fail_index=1
 	while [ $elapsed -lt $duration ]; do
 		if ! check_for_process $clients dbench; then
 			error_noexit "dbench stopped on some of $clients!"
@@ -2075,18 +2073,23 @@ test_70b () {
 			break
 		fi
 		sleep 1
-		replay_barrier $SINGLEMDS
+		replay_barrier mds$fail_index
 		sleep 1 # give clients a time to do operations
 		# Increment the number of failovers
 		num_failovers=$((num_failovers+1))
-		log "$TESTNAME fail $SINGLEMDS $num_failovers times"
-		fail $SINGLEMDS
+		log "$TESTNAME fail mds$fail_index $num_failovers times"
+		fail mds$fail_index
 		elapsed=$(($(date +%s) - start_ts))
+		if [ $fail_index -ge $MDSCOUNT ]; then
+			fail_index=1
+		else
+			fail_index=$((fail_index+1))
+		fi
 	done
 
 	wait $pid || error "rundbench load on $clients failed!"
 }
-run_test 70b "mds recovery; $CLIENTCOUNT clients"
+run_test 70b "dbench ${MDSCOUNT}mdts recovery; $CLIENTCOUNT clients"
 # end multi-client tests
 
 test_73a() {
@@ -2172,6 +2175,7 @@ test_80a() {
 	$LFS mkdir -i $MDTIDX $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
 	fail mds${MDTIDX}
 
 	wait $CLIENT_PID || error "remote creation failed"
@@ -2199,6 +2203,8 @@ test_80b() {
 	$LFS mkdir -i $MDTIDX $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "remote creation failed"
@@ -2226,6 +2232,8 @@ test_80c() {
 	$LFS mkdir -i $MDTIDX $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX}
 	fail mds$((MDTIDX + 1))
 
@@ -2252,6 +2260,9 @@ test_80d() {
 	# sleep 3 seconds to make sure MDTs are failed after
 	# lfs mkdir -i has finished on all of MDTs.
 	sleep 3
+
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX},mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "remote creation failed"
@@ -2283,6 +2294,7 @@ test_80e() {
 	# lfs mkdir -i has finished on all of MDTs.
 	sleep 3
 
+	replay_barrier mds1
 	fail mds${MDTIDX}
 
 	wait $CLIENT_PID || error "remote creation failed"
@@ -2309,6 +2321,7 @@ test_80f() {
 	$LFS mkdir -i $MDTIDX $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds2
 	fail mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "remote creation failed"
@@ -2340,6 +2353,8 @@ test_80g() {
 	# lfs mkdir -i has finished on all of MDTs.
 	sleep 3
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX}
 	fail mds$((MDTIDX + 1))
 
@@ -2367,6 +2382,8 @@ test_80h() {
 	# lfs mkdir -i has finished on all of MDTs.
 	sleep 3
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX},mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "remote dir creation failed"
@@ -2397,6 +2414,7 @@ test_81a() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds2
 	fail mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -2426,6 +2444,7 @@ test_81b() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
 	fail mds${MDTIDX}
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -2456,6 +2475,8 @@ test_81c() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX}
 	fail mds$((MDTIDX + 1))
 
@@ -2482,6 +2503,8 @@ test_81d() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX},mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -2513,6 +2536,7 @@ test_81e() {
 	local CLIENT_PID=$!
 	do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
+	replay_barrier mds1
 	fail mds${MDTIDX}
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -2543,6 +2567,7 @@ test_81f() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds2
 	fail mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -2573,6 +2598,8 @@ test_81g() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX}
 	fail mds$((MDTIDX + 1))
 
@@ -2599,6 +2626,8 @@ test_81h() {
 	rmdir $remote_dir &
 	local CLIENT_PID=$!
 
+	replay_barrier mds1
+	replay_barrier mds2
 	fail mds${MDTIDX},mds$((MDTIDX + 1))
 
 	wait $CLIENT_PID || error "rm remote dir failed"
@@ -3059,6 +3088,710 @@ test_101() { #LU-5648
 }
 run_test 101 "Shouldn't reassign precreated objs to other files after recovery"
 
+check_striped_dir_110()
+{
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+			error "create striped dir failed"
+	local stripe_count=$($LFS getdirstripe -c $DIR/$tdir/striped_dir)
+	[ $stripe_count -eq $MDSCOUNT ] ||
+		error "$stripe_count != 2 after recovery"
+}
+
+test_110a() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds1
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	fail mds1
+
+	check_striped_dir_110 || error "check striped_dir failed"
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110a "DNE: create striped dir, fail MDT1"
+
+test_110b() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds1
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	umount $MOUNT
+	fail mds1
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110b "DNE: create striped dir, fail MDT1 and client"
+
+test_110c() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds2
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	fail mds2
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110c "DNE: create striped dir, fail MDT2"
+
+test_110d() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds2
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	umount $MOUNT
+	fail mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110d "DNE: create striped dir, fail MDT2 and client"
+
+test_110e() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds2
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	umount $MOUNT
+	replay_barrier mds1
+	fail mds1,mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110e "DNE: create striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
+
+test_110f() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds1
+	replay_barrier mds2
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	fail mds1,mds2
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110f "DNE: create striped dir, fail MDT1/MDT2"
+
+test_110g() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds1
+	$LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+	umount $MOUNT
+	replay_barrier mds2
+	fail mds1,mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+
+	check_striped_dir_110 || error "check striped_dir failed"
+
+	rm -rf $DIR/$tdir || error "rmdir failed"
+
+	return 0
+}
+run_test 110g "DNE: create striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
+
+test_111a() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds1
+	rm -rf $DIR/$tdir/striped_dir
+	fail mds1
+
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111a "DNE: unlink striped dir, fail MDT1"
+
+test_111b() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds2
+	rm -rf $DIR/$tdir/striped_dir
+	umount $MOUNT
+	fail mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111b "DNE: unlink striped dir, fail MDT2"
+
+test_111c() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds1
+	rm -rf $DIR/$tdir/striped_dir
+	umount $MOUNT
+	replay_barrier mds2
+	fail mds1,mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111c "DNE: unlink striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
+
+test_111d() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds2
+	rm -rf $DIR/$tdir/striped_dir
+	umount $MOUNT
+	replay_barrier mds1
+	fail mds1,mds2
+	zconf_mount $(hostname) $MOUNT
+	client_up || return 1
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+
+	return 0
+}
+run_test 111d "DNE: unlink striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
+
+test_111e() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds2
+	rm -rf $DIR/$tdir/striped_dir
+	replay_barrier mds1
+	fail mds1,mds2
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111e "DNE: unlink striped dir, uncommit on MDT2, fail MDT1/MDT2"
+
+test_111f() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds1
+	rm -rf $DIR/$tdir/striped_dir
+	replay_barrier mds2
+	fail mds1,mds2
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111f "DNE: unlink striped dir, uncommit on MDT1, fail MDT1/MDT2"
+
+test_111g() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	mkdir -p $DIR/$tdir
+	$LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+	replay_barrier mds1
+	replay_barrier mds2
+	rm -rf $DIR/$tdir/striped_dir
+	fail mds1,mds2
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+			error "striped dir still exists"
+	return 0
+}
+run_test 111g "DNE: unlink striped dir, fail MDT1/MDT2"
+
+test_112_rename_prepare() {
+	mkdir -p $DIR/$tdir/src_dir
+	$LFS mkdir -i 1 $DIR/$tdir/src_dir/src_child ||
+		error "create remote source failed"
+
+	touch $DIR/$tdir/src_dir/src_child/a
+
+	$LFS mkdir -i 2 $DIR/$tdir/tgt_dir ||
+		error "create remote target dir failed"
+
+	$LFS mkdir -i 3 $DIR/$tdir/tgt_dir/tgt_child ||
+		error "create remote target child failed"
+}
+
+test_112_check() {
+	find $DIR/$tdir/
+	$CHECKSTAT -t dir $DIR/$tdir/src_dir/src_child &&
+		error "src_child still exists after rename"
+
+	$CHECKSTAT -t file $DIR/$tdir/tgt_dir/tgt_child/a ||
+		error "missing file(a) after rename"
+}
+
+test_112a() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+	fail mds1
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112a "DNE: cross MDT rename, fail MDT1"
+
+test_112b() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds2
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds2
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112b "DNE: cross MDT rename, fail MDT2"
+
+test_112c() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds3
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds3
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112c "DNE: cross MDT rename, fail MDT3"
+
+test_112d() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112d "DNE: cross MDT rename, fail MDT4"
+
+test_112e() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds2
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds2
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112e "DNE: cross MDT rename, fail MDT1 and MDT2"
+
+test_112f() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds3
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds3
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112f "DNE: cross MDT rename, fail MDT1 and MDT3"
+
+test_112g() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112g "DNE: cross MDT rename, fail MDT1 and MDT4"
+
+test_112h() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds2
+	replay_barrier mds3
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds2,mds3
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112h "DNE: cross MDT rename, fail MDT2 and MDT3"
+
+test_112i() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds2
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds2,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112i "DNE: cross MDT rename, fail MDT2 and MDT4"
+
+test_112j() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds3
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds3,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112j "DNE: cross MDT rename, fail MDT3 and MDT4"
+
+test_112k() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds2
+	replay_barrier mds3
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds2,mds3
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112k "DNE: cross MDT rename, fail MDT1,MDT2,MDT3"
+
+test_112l() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds2
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds2,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112l "DNE: cross MDT rename, fail MDT1,MDT2,MDT4"
+
+test_112m() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds1
+	replay_barrier mds3
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds1,mds3,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112m "DNE: cross MDT rename, fail MDT1,MDT3,MDT4"
+
+test_112n() {
+	[ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+
+	test_112_rename_prepare
+	replay_barrier mds2
+	replay_barrier mds3
+	replay_barrier mds4
+
+	mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+		error "rename dir cross MDT failed!"
+
+	fail mds2,mds3,mds4
+
+	test_112_check
+	rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112n "DNE: cross MDT rename, fail MDT2,MDT3,MDT4"
+
+test_115() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+	local fail_index=0
+	local index
+	local i
+	local j
+
+	mkdir -p $DIR/$tdir
+	for ((j=0;j<$((MDSCOUNT));j++)); do
+		fail_index=$((fail_index+1))
+		index=$((fail_index % MDSCOUNT))
+		replay_barrier mds$((index + 1))
+		for ((i=0;i<5;i++)); do
+			test_mkdir -i$index -c$MDSCOUNT $DIR/$tdir/test_$i ||
+				error "create striped dir $DIR/$tdir/test_$i"
+		done
+
+		fail mds$((index + 1))
+		for ((i=0;i<5;i++)); do
+			checkstat -t dir $DIR/$tdir/test_$i ||
+				error "$DIR/$tdir/test_$i does not exist!"
+		done
+		rm -rf $DIR/$tdir/test_* ||
+				error "rmdir fails"
+	done
+}
+run_test 115 "failover for create/unlink striped directory"
+
+test_116a() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.55) ] &&
+		skip "Do not support large update log before 2.7.55" &&
+		return 0
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+	local fail_index=0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds1
+
+	# OBD_FAIL_SPLIT_UPDATE_REC       0x1702
+	do_facet mds1 "lctl set_param fail_loc=0x80001702"
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+
+	fail mds1
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+		error "stried_dir does not exists"
+}
+run_test 116a "large update log master MDT recovery"
+
+test_116b() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.55) ] &&
+		skip "Do not support large update log before 2.7.55" &&
+		return 0
+
+	([ $FAILURE_MODE == "HARD" ] &&
+		[ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+		skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+		return 0
+	local fail_index=0
+
+	mkdir -p $DIR/$tdir
+	replay_barrier mds2
+
+	# OBD_FAIL_SPLIT_UPDATE_REC       0x1702
+	do_facet mds2 "lctl set_param fail_loc=0x80001702"
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+
+	fail mds2
+	$CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+		error "stried_dir does not exists"
+}
+run_test 116b "large update log slave MDT recovery"
+
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status