+test_93a() {
+ local server_version=$(lustre_version_code $SINGLEMDS)
+ [[ $server_version -ge $(version_code 2.6.90) ]] ||
+ [[ $server_version -ge $(version_code 2.5.4) &&
+ $server_version -lt $(version_code 2.5.50) ]] ||
+ { skip "Need MDS version 2.5.4+ or 2.6.90+"; return; }
+
+ cancel_lru_locks osc
+
+ $SETSTRIPE -i 0 -c 1 $DIR/$tfile ||
+ error "$SETSTRIPE $DIR/$tfile failed"
+ dd if=/dev/zero of=$DIR/$tfile bs=1024 count=1 ||
+ error "dd to $DIR/$tfile failed"
+ #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
+ # We need to emulate a state that OST is waiting for other clients
+ # not completing the recovery. Final ping is queued, but reply will be
+ # sent on the recovery completion. It is done by sleep before
+ # processing final pings
+ do_facet ost1 "$LCTL set_param fail_val=40"
+ do_facet ost1 "$LCTL set_param fail_loc=0x715"
+ fail ost1
+}
+run_test 93a "replay + reconnect"
+
+test_93b() {
+ local server_version=$(lustre_version_code $SINGLEMDS)
+ [[ $server_version -ge $(version_code 2.7.90) ]] ||
+ { skip "Need MDS version 2.7.90+"; return; }
+
+ cancel_lru_locks mdc
+
+ createmany -o $DIR/$tfile 20 ||
+ error "createmany -o $DIR/$tfile failed"
+
+ #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
+ # We need to emulate a state that MDT is waiting for other clients
+ # not completing the recovery. Final ping is queued, but reply will be
+ # sent on the recovery completion. It is done by sleep before
+ # processing final pings
+ do_facet mds1 "$LCTL set_param fail_val=80"
+ do_facet mds1 "$LCTL set_param fail_loc=0x715"
+ fail mds1
+}
+run_test 93b "replay + reconnect on mds"
+
+striped_dir_check_100() {
+ local striped_dir=$DIR/$tdir/striped_dir
+ local stripe_count=$($LFS getdirstripe -c $striped_dir)
+
+ $LFS getdirstripe $striped_dir
+ [ $stripe_count -eq 2 ] || error "$stripe_count != 2"
+
+ createmany -o $striped_dir/f-%d 20 ||
+ error "creation failed under striped dir"
+}
+
+test_100a() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ local striped_dir=$DIR/$tdir/striped_dir
+ local MDTIDX=1
+
+ mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+
+ #To make sure MDT1 and MDT0 are connected
+ #otherwise it may create single stripe dir here
+ $LFS setdirstripe -i1 $DIR/$tdir/remote_dir
+
+ #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
+ do_facet mds$((MDTIDX+1)) lctl set_param fail_loc=0x1701
+ $LFS setdirstripe -i0 -c2 $striped_dir &
+ local CLIENT_PID=$!
+
+ fail mds$((MDTIDX + 1))
+
+ wait $CLIENT_PID || error "striped dir creation failed"
+
+ striped_dir_check_100 || error "striped dir check failed"
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 100a "DNE: create striped dir, drop update rep from MDT1, fail MDT1"
+
+test_100b() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ local striped_dir=$DIR/$tdir/striped_dir
+ local MDTIDX=1
+
+ mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+
+ #To make sure MDT1 and MDT0 are connected
+ #otherwise it may create single stripe dir here
+ $LFS setdirstripe -i1 $DIR/$tdir/remote_dir
+
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds$MDTIDX lctl set_param fail_loc=0x119
+ $LFS mkdir -i0 -c2 $striped_dir &
+
+ local CLIENT_PID=$!
+ fail mds$MDTIDX
+
+ wait $CLIENT_PID || error "striped dir creation failed"
+
+ striped_dir_check_100 || error "striped dir check failed"
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 100b "DNE: create striped dir, fail MDT0"
+
+test_101() { #LU-5648
+ mkdir -p $DIR/$tdir/d1
+ mkdir -p $DIR/$tdir/d2
+ touch $DIR/$tdir/file0
+ num=1000
+
+ replay_barrier $SINGLEMDS
+ for i in $(seq $num) ; do
+ echo test$i > $DIR/$tdir/d1/file$i
+ done
+
+ fail_abort $SINGLEMDS
+ for i in $(seq $num) ; do
+ touch $DIR/$tdir/d2/file$i
+ test -s $DIR/$tdir/d2/file$i &&
+ ls -al $DIR/$tdir/d2/file$i && error "file$i's size > 0"
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 101 "Shouldn't reassign precreated objs to other files after recovery"
+
+test_102a() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop request on MDT to force resend
+ #define OBD_FAIL_MDS_REINT_MULTI_NET 0x159
+ do_facet $facet "$LCTL set_param fail_loc=0x159"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102a "check resend (request lost) with multiple modify RPCs in flight"
+
+test_102b() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop reply on MDT to force reconstruction
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102b "check resend (reply lost) with multiple modify RPCs in flight"
+
+test_102c() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ replay_barrier $facet
+
+ # drop reply on MDT
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+ do_facet $facet "$LCTL set_param fail_loc=0"
+
+ # fail MDT
+ fail $facet
+
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102c "check replay w/o reconstruction with multiple mod RPCs in flight"
+
+test_102d() {
+ local idx
+ local facet
+ local num
+ local i
+ local pids pid
+
+ [[ $(lctl get_param mdc.*.import |
+ grep "connect_flags:.*multi_mod_rpc") ]] ||
+ { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
+
+ $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+ idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
+ facet="mds$((0x$idx + 1))"
+
+ # get current value of max_mod_rcps_in_flight
+ num=$($LCTL get_param -n \
+ mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
+ # set default value if client does not support multi mod RPCs
+ [ -z "$num" ] && num=1
+
+ echo "creating $num files ..."
+ umask 0022
+ for i in $(seq $num); do
+ touch $DIR/$tdir/file-$i
+ done
+
+ # drop reply on MDT
+ #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
+ do_facet $facet "$LCTL set_param fail_loc=0x15a"
+ echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
+ for i in $(seq $num); do
+ chmod 0600 $DIR/$tdir/file-$i &
+ pids="$pids $!"
+ done
+ sleep 1
+
+ # write MDT transactions to disk
+ do_facet $facet "sync; sync; sync"
+
+ do_facet $facet "$LCTL set_param fail_loc=0"
+
+ # fail MDT
+ fail $facet
+
+ for pid in $pids; do
+ wait $pid || error "chmod failed"
+ done
+ echo "done ($(date +%H:%M:%S))"
+
+ # check chmod succeed
+ for i in $(seq $num); do
+ checkstat -vp 0600 $DIR/$tdir/file-$i
+ done
+
+ rm -rf $DIR/$tdir
+}
+run_test 102d "check replay & reconstruction with multiple mod RPCs in flight"
+
+test_103() {
+ remote_mds_nodsh && skip "remote MDS with nodsh" && return
+#define OBD_FAIL_MDS_TRACK_OVERFLOW 0x162
+ do_facet mds1 $LCTL set_param fail_loc=0x80000162
+
+ mkdir -p $DIR/$tdir
+ createmany -o $DIR/$tdir/t- 30 ||
+ error "create files on remote directory failed"
+ sync
+ rm -rf $DIR/$tdir/t-*
+ sync
+#MDS should crash with tr->otr_next_id overflow
+ fail mds1
+}
+run_test 103 "Check otr_next_id overflow"
+
+
+check_striped_dir_110()
+{
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+ error "create striped dir failed"
+ local stripe_count=$($LFS getdirstripe -c $DIR/$tdir/striped_dir)
+ [ $stripe_count -eq $MDSCOUNT ] ||
+ error "$stripe_count != 2 after recovery"
+}
+
+test_110a() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds1
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ fail mds1
+
+ check_striped_dir_110 || error "check striped_dir failed"
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110a "DNE: create striped dir, fail MDT1"
+
+test_110b() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds1
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ umount $MOUNT
+ fail mds1
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110b "DNE: create striped dir, fail MDT1 and client"
+
+test_110c() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds2
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ fail mds2
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110c "DNE: create striped dir, fail MDT2"
+
+test_110d() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds2
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ umount $MOUNT
+ fail mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110d "DNE: create striped dir, fail MDT2 and client"
+
+test_110e() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds2
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ umount $MOUNT
+ replay_barrier mds1
+ fail mds1,mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110e "DNE: create striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
+
+test_110f() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds1
+ replay_barrier mds2
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ fail mds1,mds2
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110f "DNE: create striped dir, fail MDT1/MDT2"
+
+test_110g() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds1
+ $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
+ umount $MOUNT
+ replay_barrier mds2
+ fail mds1,mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+
+ check_striped_dir_110 || error "check striped_dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110g "DNE: create striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
+
+test_111a() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds1
+ rm -rf $DIR/$tdir/striped_dir
+ fail mds1
+
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111a "DNE: unlink striped dir, fail MDT1"
+
+test_111b() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds2
+ rm -rf $DIR/$tdir/striped_dir
+ umount $MOUNT
+ fail mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111b "DNE: unlink striped dir, fail MDT2"
+
+test_111c() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds1
+ rm -rf $DIR/$tdir/striped_dir
+ umount $MOUNT
+ replay_barrier mds2
+ fail mds1,mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111c "DNE: unlink striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
+
+test_111d() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds2
+ rm -rf $DIR/$tdir/striped_dir
+ umount $MOUNT
+ replay_barrier mds1
+ fail mds1,mds2
+ zconf_mount $(hostname) $MOUNT
+ client_up || return 1
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+
+ return 0
+}
+run_test 111d "DNE: unlink striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
+
+test_111e() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds2
+ rm -rf $DIR/$tdir/striped_dir
+ replay_barrier mds1
+ fail mds1,mds2
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111e "DNE: unlink striped dir, uncommit on MDT2, fail MDT1/MDT2"
+
+test_111f() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds1
+ rm -rf $DIR/$tdir/striped_dir
+ replay_barrier mds2
+ fail mds1,mds2
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111f "DNE: unlink striped dir, uncommit on MDT1, fail MDT1/MDT2"
+
+test_111g() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
+ replay_barrier mds1
+ replay_barrier mds2
+ rm -rf $DIR/$tdir/striped_dir
+ fail mds1,mds2
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
+ error "striped dir still exists"
+ return 0
+}
+run_test 111g "DNE: unlink striped dir, fail MDT1/MDT2"
+
+test_112_rename_prepare() {
+ mkdir -p $DIR/$tdir/src_dir
+ $LFS mkdir -i 1 $DIR/$tdir/src_dir/src_child ||
+ error "create remote source failed"
+
+ touch $DIR/$tdir/src_dir/src_child/a
+
+ $LFS mkdir -i 2 $DIR/$tdir/tgt_dir ||
+ error "create remote target dir failed"
+
+ $LFS mkdir -i 3 $DIR/$tdir/tgt_dir/tgt_child ||
+ error "create remote target child failed"
+}
+
+test_112_check() {
+ find $DIR/$tdir/
+ $CHECKSTAT -t dir $DIR/$tdir/src_dir/src_child &&
+ error "src_child still exists after rename"
+
+ $CHECKSTAT -t file $DIR/$tdir/tgt_dir/tgt_child/a ||
+ error "missing file(a) after rename"
+}
+
+test_112a() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+ fail mds1
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112a "DNE: cross MDT rename, fail MDT1"
+
+test_112b() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds2
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds2
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112b "DNE: cross MDT rename, fail MDT2"
+
+test_112c() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds3
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds3
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112c "DNE: cross MDT rename, fail MDT3"
+
+test_112d() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112d "DNE: cross MDT rename, fail MDT4"
+
+test_112e() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds2
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds2
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112e "DNE: cross MDT rename, fail MDT1 and MDT2"
+
+test_112f() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds3
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds3
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112f "DNE: cross MDT rename, fail MDT1 and MDT3"
+
+test_112g() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112g "DNE: cross MDT rename, fail MDT1 and MDT4"
+
+test_112h() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds2
+ replay_barrier mds3
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds2,mds3
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112h "DNE: cross MDT rename, fail MDT2 and MDT3"
+
+test_112i() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds2
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds2,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112i "DNE: cross MDT rename, fail MDT2 and MDT4"
+
+test_112j() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds3
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds3,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112j "DNE: cross MDT rename, fail MDT3 and MDT4"
+
+test_112k() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds2
+ replay_barrier mds3
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds2,mds3
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112k "DNE: cross MDT rename, fail MDT1,MDT2,MDT3"
+
+test_112l() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds2
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds2,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112l "DNE: cross MDT rename, fail MDT1,MDT2,MDT4"
+
+test_112m() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds1
+ replay_barrier mds3
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds1,mds3,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112m "DNE: cross MDT rename, fail MDT1,MDT3,MDT4"
+
+test_112n() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ test_112_rename_prepare
+ replay_barrier mds2
+ replay_barrier mds3
+ replay_barrier mds4
+
+ mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
+ error "rename dir cross MDT failed!"
+
+ fail mds2,mds3,mds4
+
+ test_112_check
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 112n "DNE: cross MDT rename, fail MDT2,MDT3,MDT4"
+
+test_115() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local fail_index=0
+ local index
+ local i
+ local j
+
+ mkdir -p $DIR/$tdir
+ for ((j=0;j<$((MDSCOUNT));j++)); do
+ fail_index=$((fail_index+1))
+ index=$((fail_index % MDSCOUNT))
+ replay_barrier mds$((index + 1))
+ for ((i=0;i<5;i++)); do
+ test_mkdir -i$index -c$MDSCOUNT $DIR/$tdir/test_$i ||
+ error "create striped dir $DIR/$tdir/test_$i"
+ done
+
+ fail mds$((index + 1))
+ for ((i=0;i<5;i++)); do
+ checkstat -t dir $DIR/$tdir/test_$i ||
+ error "$DIR/$tdir/test_$i does not exist!"
+ done
+ rm -rf $DIR/$tdir/test_* ||
+ error "rmdir fails"
+ done
+}
+run_test 115 "failover for create/unlink striped directory"
+
+test_116a() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.55) ] &&
+ skip "Do not support large update log before 2.7.55" &&
+ return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local fail_index=0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds1
+
+ # OBD_FAIL_SPLIT_UPDATE_REC 0x1702
+ do_facet mds1 "lctl set_param fail_loc=0x80001702"
+ $LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+
+ fail mds1
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+ error "stried_dir does not exists"
+}
+run_test 116a "large update log master MDT recovery"
+
+test_116b() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.55) ] &&
+ skip "Do not support large update log before 2.7.55" &&
+ return 0
+
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local fail_index=0
+
+ mkdir -p $DIR/$tdir
+ replay_barrier mds2
+
+ # OBD_FAIL_SPLIT_UPDATE_REC 0x1702
+ do_facet mds2 "lctl set_param fail_loc=0x80001702"
+ $LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+
+ fail mds2
+ $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
+ error "stried_dir does not exists"
+}
+run_test 116b "large update log slave MDT recovery"
+
+test_117() {
+ [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local index
+ local mds_indexs
+
+ mkdir -p $DIR/$tdir
+ $LFS setdirstripe -i0 -c$MDSCOUNT $DIR/$tdir/remote_dir
+ $LFS setdirstripe -i1 -c$MDSCOUNT $DIR/$tdir/remote_dir_1
+ sleep 2
+
+ # Let's set rdonly on all MDTs, so client will send
+ # replay requests on all MDTs and replay these requests
+ # at the same time. This test will verify the recovery
+ # will not be deadlock in this case, LU-7531.
+ for ((index = 0; index < $((MDSCOUNT)); index++)); do
+ replay_barrier mds$((index + 1))
+ if [ -z $mds_indexs ]; then
+ mds_indexs="${mds_indexs}mds$((index+1))"
+ else
+ mds_indexs="${mds_indexs},mds$((index+1))"
+ fi
+ done
+
+ rm -rf $DIR/$tdir/remote_dir
+ rm -rf $DIR/$tdir/remote_dir_1
+
+ fail $mds_indexs
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 117 "DNE: cross MDT unlink, fail MDT1 and MDT2"
+
+test_118() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+
+ mkdir -p $DIR/$tdir
+
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir ||
+ error "setdirstripe fails"
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1 ||
+ error "setdirstripe fails 1"
+ rm -rf $DIR/$tdir/striped_dir* || error "rmdir fails"
+
+ # OBD_FAIL_INVALIDATE_UPDATE 0x1705
+ do_facet mds1 "lctl set_param fail_loc=0x1705"
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
+ do_facet mds1 "lctl set_param fail_loc=0x0"
+
+ replay_barrier mds1
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
+ $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
+ fail mds1
+
+ true
+}
+run_test 118 "invalidate osp update will not cause update log corruption"
+
+test_119() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+ local stripe_count
+ local hard_timeout=$(do_facet mds1 \
+ "lctl get_param -n mdt.$FSNAME-MDT0000.recovery_time_hard")
+
+ local clients=${CLIENTS:-$HOSTNAME}
+ local time_min=$(recovery_time_min)
+
+ mkdir -p $DIR/$tdir
+ mkdir $DIR/$tdir/tmp
+ rmdir $DIR/$tdir/tmp
+
+ replay_barrier mds1
+ mkdir $DIR/$tdir/dir_1
+ for ((i = 0; i < 20; i++)); do
+ $LFS setdirstripe -c2 $DIR/$tdir/stripe_dir-$i
+ done
+
+ stop mds1
+ change_active mds1
+ wait_for_facet mds1
+
+ #define OBD_FAIL_TGT_REPLAY_DELAY 0x714
+ do_facet mds1 $LCTL set_param fail_loc=0x80000714
+ #sleep (timeout + 5), so mds will evict the client exports,
+ #but DNE update recovery will keep going.
+ do_facet mds1 $LCTL set_param fail_val=$((time_min + 5))
+
+ mount_facet mds1 "-o recovery_time_hard=$time_min"
+
+ wait_clients_import_state "$clients" mds1 FULL
+
+ clients_up || clients_up || error "failover df: $?"
+
+ #revert back the hard timeout
+ do_facet mds1 $LCTL set_param \
+ mdt.$FSNAME-MDT0000.recovery_time_hard=$hard_timeout
+
+ for ((i = 0; i < 20; i++)); do
+ stripe_count=$($LFS getdirstripe -c $DIR/$tdir/stripe_dir-$i)
+ [ $stripe_count == 2 ] || {
+ error "stripe_dir-$i creation replay fails"
+ break
+ }
+ done
+}
+run_test 119 "timeout of normal replay does not cause DNE replay fails "
+
+test_120() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+ skip "Do not support large update log before 2.7.64" &&
+ return 0
+
+ mkdir $DIR/$tdir
+ replay_barrier_nosync mds1
+ for ((i = 0; i < 20; i++)); do
+ mkdir $DIR/$tdir/dir-$i || {
+ error "create dir-$i fails"
+ break
+ }
+ $LFS setdirstripe -c2 $DIR/$tdir/stripe_dir-$i || {
+ error "create stripe_dir-$i fails"
+ break
+ }
+ done
+
+ fail_abort mds1
+
+ for ((i = 0; i < 20; i++)); do
+ [ ! -e "$DIR/$tdir/dir-$i" ] || {
+ error "dir-$i still exists"
+ break
+ }
+ [ ! -e "$DIR/$tdir/stripe_dir-$i" ] || {
+ error "stripe_dir-$i still exists"
+ break
+ }
+ done
+}
+run_test 120 "DNE fail abort should stop both normal and DNE replay"
+