+test_22a () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+
+ local MDTIDX=1
+ local remote_dir=${tdir}/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+
+ fail mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "lfs mkdir failed"
+
+ replay_barrier mds$MDTIDX
+ create_remote_dir_files_22 || error "Remote creation failed $?"
+ fail mds$MDTIDX
+
+ checkstat_22 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 22a "c1 lfs mkdir -i 1 dir1, M1 drop reply & fail, c2 mkdir dir1/dir"
+
+test_22b () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local MDTIDX=1
+ local remote_dir=$tdir/remote_dir
+
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+
+ fail mds${MDTIDX},mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "lfs mkdir failed"
+
+ replay_barrier mds$MDTIDX
+ create_remote_dir_files_22 || error "Remote creation failed $?"
+ fail mds${MDTIDX}
+
+ checkstat_22 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 22b "c1 lfs mkdir -i 1 d1, M1 drop reply & fail M0/M1, c2 mkdir d1/dir"
+
+test_22c () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local MDTIDX=1
+ local remote_dir=${tdir}/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+
+ # OBD_FAIL_UPDATE_OBJ_NET_REP 0x1701
+ do_facet mds$MDTIDX lctl set_param fail_loc=0x1701
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+ do_facet mds$MDTIDX lctl set_param fail_loc=0
+
+ fail mds$MDTIDX
+ wait $CLIENT_PID || error "lfs mkdir failed"
+
+ replay_barrier mds$MDTIDX
+ create_remote_dir_files_22 || error "Remote creation failed $?"
+ fail mds$MDTIDX
+
+ checkstat_22 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 22c "c1 lfs mkdir -i 1 d1, M1 drop update & fail M1, c2 mkdir d1/dir"
+
+test_22d () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local MDTIDX=1
+ local remote_dir=${tdir}/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+
+ # OBD_FAIL_UPDATE_OBJ_NET_REP 0x1701
+ do_facet mds$MDTIDX lctl set_param fail_loc=0x1701
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+ do_facet mds$MDTIDX lctl set_param fail_loc=0
+
+ fail mds${MDTIDX},mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "lfs mkdir failed"
+
+ replay_barrier mds$MDTIDX
+ create_remote_dir_files_22 || error "Remote creation failed $?"
+ fail mds$MDTIDX
+
+ checkstat_22 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 22d "c1 lfs mkdir -i 1 d1, M1 drop update & fail M0/M1,c2 mkdir d1/dir"
+
+checkstat_23() {
+ checkstat $MOUNT1/$remote_dir || return 1
+ checkstat $MOUNT1/$remote_dir/$tfile-1 || return 2
+ return 0
+}
+
+create_remote_dir_files_23() {
+ do_node $CLIENT2 mkdir ${MOUNT2}/$remote_dir || return 1
+ do_node $CLIENT2 createmany -o $MOUNT2/$remote_dir/$tfile- 2 || return 2
+ return 0
+}
+
+test_23a () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local MDTIDX=1
+ local remote_dir=$tdir/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
+ error "lfs mkdir failed"
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
+ do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
+ local CLIENT_PID=$!
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
+
+ fail mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "rmdir remote dir failed"
+
+ replay_barrier mds${MDTIDX}
+ create_remote_dir_files_23 || error "Remote creation failed $?"
+ fail mds${MDTIDX}
+
+ checkstat_23 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 23a "c1 rmdir d1, M1 drop reply and fail, client2 mkdir d1"
+
+test_23b () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local MDTIDX=1
+ local remote_dir=$tdir/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
+ error "lfs mkdir failed"
+
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
+ do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
+ local CLIENT_PID=$!
+ do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
+
+ fail mds${MDTIDX},mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "rmdir remote dir failed"
+
+ replay_barrier mds${MDTIDX}
+ create_remote_dir_files_23 || error "Remote creation failed $?"
+ fail mds${MDTIDX}
+
+ checkstat_23 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
+ return 0
+}
+run_test 23b "c1 rmdir d1, M1 drop reply and fail M0/M1, c2 mkdir d1"
+
+test_23c () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+
+ ([ $FAILURE_MODE == "HARD" ] &&
+ [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
+ skip "MDTs needs to be on diff hosts for HARD fail mode" &&
+ return 0
+ local MDTIDX=1
+ local remote_dir=$tdir/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
+ error "lfs mkdir failed"
+
+ # OBD_FAIL_UPDATE_OBJ_NET_REP 0x1701
+ do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
+ do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+ do_facet mds${MDTIDX} lctl set_param fail_loc=0
+
+ fail mds${MDTIDX}
+ wait $CLIENT_PID || error "rmdir remote dir failed"
+
+ replay_barrier mds${MDTIDX}
+ create_remote_dir_files_23 || error "Remote creation failed $?"
+ fail mds${MDTIDX}
+
+ checkstat_23 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || return 6
+ return 0
+}
+run_test 23c "c1 rmdir d1, M0 drop update reply and fail M0, c2 mkdir d1"
+
+test_23d () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local MDTIDX=1
+ local remote_dir=$tdir/remote_dir
+
+ do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
+ do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
+ error "lfs mkdir failed"
+
+ # OBD_FAIL_UPDATE_OBJ_NET 0x1701
+ do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
+ do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
+ CLIENT_PID=$!
+ do_facet mds${MDTIDX} lctl set_param fail_loc=0
+
+ fail mds${MDTIDX},mds$((MDTIDX + 1))
+ wait $CLIENT_PID || error "rmdir remote dir failed"
+
+ replay_barrier mds${MDTIDX}
+ create_remote_dir_files_23 || error "Remote creation failed $?"
+ fail mds${MDTIDX}
+
+ checkstat_23 || error "check stat failed $?"
+
+ rm -rf $MOUNT1/$tdir || return 6
+ return 0
+}
+run_test 23d "c1 rmdir d1, M0 drop update reply and fail M0/M1, c2 mkdir d1"
+
+test_24 () {
+ [[ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.5.2) ]] ||
+ { skip "Need MDS version newer than 2.5.2"; return 0; }
+
+ touch $MOUNT/$tfile
+ stat $MOUNT/$tfile >&/dev/null
+# OBD_FAIL_MDS_REINT_NET_REP
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x119
+ $TRUNCATE $MOUNT/$tfile 100 &
+ PID=$!
+ sleep 1
+ do_facet $SINGLEMDS lctl set_param fail_loc=0
+ # sync to release rep-ack lock quickly
+ do_nodes $(comma_list $(mdts_nodes)) \
+ "lctl set_param -n osd*.*MDT*.force_sync 1"
+ rm $MOUNT2/$tfile
+ wait
+}
+run_test 24 "reconstruct on non-existing object"
+
+# end commit on sharing tests
+
+test_25() {
+ cancel_lru_locks osc
+
+ $SETSTRIPE -i 0 -c 1 $DIR/$tfile
+
+ # get lock for the 1st client
+ dd if=/dev/zero of=$DIR/$tfile count=1 >/dev/null ||
+ error "failed to write data"
+
+ # get waiting locks for the 2nd client
+ drop_ldlm_cancel "multiop $DIR2/$tfile Ow512" &
+ sleep 1
+
+ # failover, replay and resend replayed waiting locks
+ if [ $(lustre_version_code ost1) -ge $(version_code 2.6.90) ]; then
+ #define OBD_FAIL_LDLM_SRV_CP_AST 0x325
+ do_facet ost1 lctl set_param fail_loc=0x80000325
+ else
+ #define OBD_FAIL_OST_LDLM_REPLY_NET 0x213
+ do_facet ost1 lctl set_param fail_loc=0x80000213