+cleanup_106() {
+ trap 0
+ umount_client $DIR2
+ debugrestore
+}
+
+test_106() { # LU-1789
+ [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.50) ]] ||
+ { skip "Need MDS version at least 2.3.50"; return 0; }
+
+#define OBD_FAIL_MDC_LIGHTWEIGHT 0x805
+ $LCTL set_param fail_loc=0x805
+
+ debugsave
+ trap cleanup_106 EXIT
+
+ # enable lightweight flag on mdc connection
+ mount_client $DIR2
+
+ local MDS_NEXP=$(do_facet $SINGLEMDS \
+ lctl get_param -n mdt.${mds1_svc}.num_exports |
+ cut -d' ' -f2)
+ $LCTL set_param fail_loc=0
+
+ touch $DIR2/$tfile || error "failed to create empty file"
+ replay_barrier $SINGLEMDS
+
+ $LCTL set_param debug=console
+ $LCTL clear
+ facet_failover $SINGLEMDS
+
+ # lightweight connection must be evicted
+ touch -c $DIR2/$tfile || true
+ $LCTL dk $TMP/lustre-log-$TESTNAME.log
+ evicted=`awk '/This client was evicted by .*MDT0000/ {
+ print;
+ }' $TMP/lustre-log-$TESTNAME.log`
+ [ -z "$evicted" ] && error "lightweight client not evicted by mds"
+
+ # and all operations performed by lightweight client should be
+ # synchronous, so the file created before mds restart should be there
+ $CHECKSTAT -t file $DIR/$tfile || error "file not present"
+ rm -f $DIR/$tfile
+
+ cleanup_106
+}
+run_test 106 "lightweight connection support"
+
+test_107 () {
+ local CLIENT_PID
+ local close_pid
+
+ mkdir -p $DIR/$tdir
+ # OBD_FAIL_MDS_REINT_NET_REP 0x119
+ do_facet $SINGLEMDS lctl set_param fail_loc=0x119
+ multiop $DIR/$tdir D_c &
+ close_pid=$!
+ mkdir $DIR/$tdir/dir_106 &
+ CLIENT_PID=$!
+ do_facet $SINGLEMDS lctl set_param fail_loc=0
+ fail $SINGLEMDS
+
+ wait $CLIENT_PID || rc=$?
+ checkstat -t dir $DIR/$tdir/dir_106 || return 1
+
+ kill -USR1 $close_pid
+ wait $close_pid || return 2
+
+ return $rc
+}
+run_test 107 "drop reint reply, then restart MDT"
+
+test_108() {
+ mkdir -p $DIR/$tdir
+ $SETSTRIPE -c 1 -i 0 $DIR/$tdir
+
+ dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=256 &
+ local dd_pid=$!
+ sleep 0.1
+
+ ost_evict_client
+
+ wait $dd_pid
+
+ client_up || error "reconnect failed"
+ rm -f $DIR/$tdir/$tfile
+}
+run_test 108 "client eviction don't crash"
+
+test_110a () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+ local num
+
+ #prepare for 110 test, which need set striped dir on remote MDT.
+ for num in $(seq $MDSCOUNT); do
+ do_facet mds$num \
+ lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \
+ 2>/dev/null
+ done
+
+ mkdir -p $DIR/$tdir
+ drop_request "$LFS mkdir -i $MDTIDX -c2 $remote_dir" ||
+ error "lfs mkdir failed"
+ local diridx=$($GETSTRIPE -M $remote_dir)
+ [ $diridx -eq $MDTIDX ] || error "$diridx != $MDTIDX"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 110a "create remote directory: drop client req"
+
+test_110b () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ drop_reint_reply "$LFS mkdir -i $MDTIDX -c2 $remote_dir" ||
+ error "lfs mkdir failed"
+
+ diridx=$($GETSTRIPE -M $remote_dir)
+ [ $diridx -eq $MDTIDX ] || error "$diridx != $MDTIDX"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 110b "create remote directory: drop Master rep"
+
+test_110c () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ drop_update_reply $MDTIDX "$LFS mkdir -i $MDTIDX -c2 $remote_dir" ||
+ error "lfs mkdir failed"
+
+ diridx=$($GETSTRIPE -M $remote_dir)
+ [ $diridx -eq $MDTIDX ] || error "$diridx != $MDTIDX"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 110c "create remote directory: drop update rep on slave MDT"
+
+test_110d () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i $MDTIDX -c2 $remote_dir || error "lfs mkdir failed"
+
+ drop_request "rm -rf $remote_dir" || error "rm remote dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110d "remove remote directory: drop client req"
+
+test_110e () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i $MDTIDX -c2 $remote_dir || error "lfs mkdir failed"
+ drop_reint_reply "rm -rf $remote_dir" || error "rm remote dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+
+ return 0
+}
+run_test 110e "remove remote directory: drop master rep"
+
+test_110f () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ $LFS mkdir -i $MDTIDX -c2 $remote_dir || error "lfs mkdir failed"
+ drop_update_reply $MDTIDX "rm -rf $remote_dir" ||
+ error "rm remote dir failed"
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 110f "remove remote directory: drop slave rep"
+
+test_110g () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local MDTIDX=1
+
+ mkdir -p $remote_dir
+
+ createmany -o $remote_dir/f 100
+
+ #define OBD_FAIL_MIGRATE_NET_REP 0x1702
+ do_facet mds$MDTIDX lctl set_param fail_loc=0x1702
+ $LFS migrate -m $MDTIDX $remote_dir || error "migrate failed"
+ do_facet mds$MDTIDX lctl set_param fail_loc=0x0
+
+ for file in $(find $remote_dir); do
+ mdt_index=$($LFS getstripe -M $file)
+ [ $mdt_index == $MDTIDX ] ||
+ error "$file is not on MDT${MDTIDX}"
+ done
+
+ rm -rf $DIR/$tdir || error "rmdir failed"
+}
+run_test 110g "drop reply during migration"
+
+test_110h () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local src_dir=$DIR/$tdir/source_dir
+ local tgt_dir=$DIR/$tdir/target_dir
+ local MDTIDX=1
+
+ mkdir -p $src_dir
+ $LFS mkdir -i $MDTIDX $tgt_dir
+
+ dd if=/etc/hosts of=$src_dir/src_file
+ touch $tgt_dir/tgt_file
+ drop_update_reply $MDTIDX \
+ "mrename $src_dir/src_file $tgt_dir/tgt_file" ||
+ error "mrename failed"
+
+ $CHECKSTAT -t file $src_dir/src_file &&
+ error "src_file present after rename"
+
+ diff /etc/hosts $tgt_dir/tgt_file ||
+ error "file changed after rename"
+
+}
+run_test 110h "drop update reply during cross-MDT file rename"
+
+test_110i () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local src_dir=$DIR/$tdir/source_dir
+ local tgt_dir=$DIR/$tdir/target_dir
+ local MDTIDX=1
+
+ mkdir -p $src_dir
+ $LFS mkdir -i $MDTIDX $tgt_dir
+
+ mkdir $src_dir/src_dir
+ touch $src_dir/src_dir/a
+ mkdir $tgt_dir/tgt_dir
+ drop_update_reply $MDTIDX \
+ "mrename $src_dir/src_dir $tgt_dir/tgt_dir" ||
+ error "mrename failed"
+
+ $CHECKSTAT -t dir $src_dir/src_dir &&
+ error "src_dir present after rename"
+
+ $CHECKSTAT -t dir $tgt_dir/tgt_dir ||
+ error "tgt_dir not present after rename"
+
+ $CHECKSTAT -t file $tgt_dir/tgt_dir/a ||
+ error "a not present after rename"
+}
+run_test 110i "drop update reply during cross-MDT dir rename"
+
+test_110j () {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+ local remote_dir=$DIR/$tdir/remote_dir
+ local local_dir=$DIR/$tdir/local_dir
+ local MDTIDX=1
+
+ mkdir -p $DIR/$tdir
+ mkdir $DIR/$tdir/local_dir
+ $LFS mkdir -i $MDTIDX $remote_dir
+
+ touch $local_dir/local_file
+ drop_update_reply $MDTIDX \
+ "ln $local_dir/local_file $remote_dir/remote_file" ||
+ error "ln failed"
+
+ $CHECKSTAT -t file $remote_dir/remote_file ||
+ error "remote not present after ln"
+}
+run_test 110j "drop update reply during cross-MDT ln"
+
+# LU-2844 mdt prepare fail should not cause umount oops
+test_111 ()
+{
+ [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.62) ]] ||
+ { skip "Need MDS version at least 2.3.62"; return 0; }
+
+ local mdsdev=$(mdsdevname ${SINGLEMDS//mds/})
+#define OBD_FAIL_MDS_CHANGELOG_INIT 0x151
+ do_facet $SINGLEMDS lctl set_param fail_loc=0x151
+ stop $SINGLEMDS || error "stop MDS failed"
+ start $SINGLEMDS $mdsdev && error "start MDS should fail"
+ do_facet $SINGLEMDS lctl set_param fail_loc=0
+ start $SINGLEMDS $mdsdev || error "start MDS failed"
+}
+run_test 111 "mdd setup fail should not cause umount oops"
+
+# LU-793
+test_112a() {
+ remote_ost_nodsh && skip "remote OST with nodsh" && return 0
+
+ do_facet_random_file client $TMP/$tfile 100K ||
+ error_noexit "Create random file $TMP/$tfile"
+
+ pause_bulk "cp $TMP/$tfile $DIR/$tfile" $TIMEOUT ||
+ error_noexit "Can't pause_bulk copy"
+
+ df $DIR
+ # expect cmp to succeed, client resent bulk
+ cmp $TMP/$tfile $DIR/$tfile ||
+ error_noexit "Wrong data has been written"
+ rm $DIR/$tfile ||
+ error_noexit "Can't remove file"
+ rm $TMP/$tfile
+}
+run_test 112a "bulk resend while orignal request is in progress"
+
+# parameters: fail_loc CMD RC
+test_120_reply() {
+ local PID
+ local PID2
+ local rc=5
+ local fail
+
+ #define OBD_FAIL_LDLM_CP_CB_WAIT2 0x320
+ #define OBD_FAIL_LDLM_CP_CB_WAIT3 0x321
+ #define OBD_FAIL_LDLM_CP_CB_WAIT4 0x322
+ #define OBD_FAIL_LDLM_CP_CB_WAIT5 0x323
+
+ echo
+ echo -n "** FLOCK REPLY vs. EVICTION race, lock $2"
+ [ "$1" = "CLEANUP" ] &&
+ fail=0x80000320 && echo ", $1 cp first"
+ [ "$1" = "REPLY" ] &&
+ fail=0x80000321 && echo ", $1 cp first"
+ [ "$1" = "DEADLOCK CLEANUP" ] &&
+ fail=0x80000322 && echo " DEADLOCK, CLEANUP cp first"
+ [ "$1" = "DEADLOCK REPLY" ] &&
+ fail=0x80000323 && echo " DEADLOCK, REPLY cp first"
+
+ if [ x"$2" = x"get" ]; then
+ #for TEST lock, take a conflict in advance
+ # sleep longer than evictor to not confuse fail_loc: 2+2+4
+ echo "** Taking conflict **"
+ flocks_test 5 set read sleep 10 $DIR/$tfile &
+ PID2=$!
+
+ sleep 2
+ fi
+
+ $LCTL set_param fail_loc=$fail
+
+ flocks_test 5 $2 write $DIR/$tfile &
+ PID=$!
+
+ sleep 2
+ echo "** Evicting and re-connecting client **"
+ mds_evict_client
+
+ client_reconnect
+
+ if [ x"$2" = x"get" ]; then
+ wait $PID2
+ fi
+
+ wait $PID
+ rc=$?
+
+ # check if the return value is allowed
+ [ $rc -eq $3 ] && rc=0
+
+ $LCTL set_param fail_loc=0
+ return $rc
+}
+
+# a lock is taken, unlock vs. cleanup_resource() race for destroying
+# the ORIGINAL lock.
+test_120_destroy()
+{
+ local PID
+
+ flocks_test 5 set write sleep 4 $DIR/$tfile &
+ PID=$!
+ sleep 2
+
+ # let unlock to sleep in CP CB
+ $LCTL set_param fail_loc=$1
+ sleep 4
+
+ # let cleanup to cleep in CP CB
+ mds_evict_client
+
+ client_reconnect
+
+ wait $PID
+ rc=$?
+
+ $LCTL set_param fail_loc=0
+ return $rc
+}
+
+test_120() {
+ flock_is_enabled || { skip "mount w/o flock enabled" && return; }
+ touch $DIR/$tfile
+
+ test_120_reply "CLEANUP" set 5 || error "SET race failed"
+ test_120_reply "CLEANUP" get 5 || error "GET race failed"
+ test_120_reply "CLEANUP" unlock 5 || error "UNLOCK race failed"
+
+ test_120_reply "REPLY" set 5 || error "SET race failed"
+ test_120_reply "REPLY" get 5 || error "GET race failed"
+ test_120_reply "REPLY" unlock 5 || error "UNLOCK race failed"
+
+ # DEADLOCK tests
+ test_120_reply "DEADLOCK CLEANUP" set 5 || error "DEADLOCK race failed"
+ test_120_reply "DEADLOCK REPLY" set 35 || error "DEADLOCK race failed"
+
+ test_120_destroy 0x320 || error "unlock-cleanup race failed"
+}
+run_test 120 "flock race: completion vs. evict"
+
+test_113() {
+ local BEFORE=$(date +%s)
+ local EVICT
+
+ # modify dir so that next revalidate would not obtain UPDATE lock
+ touch $DIR
+
+ # drop 1 reply with UPDATE lock,
+ # resend should not create 2nd lock on server
+ mcreate $DIR/$tfile || error "mcreate failed: $?"
+ drop_ldlm_reply_once "stat $DIR/$tfile" || error "stat failed: $?"
+
+ # 2 BL AST will be sent to client, both must find the same lock,
+ # race them to not get EINVAL for 2nd BL AST
+ #define OBD_FAIL_LDLM_PAUSE_CANCEL2 0x31f
+ $LCTL set_param fail_loc=0x8000031f
+
+ $LCTL set_param ldlm.namespaces.*.early_lock_cancel=0 > /dev/null
+ chmod 0777 $DIR/$tfile || error "chmod failed: $?"
+ $LCTL set_param ldlm.namespaces.*.early_lock_cancel=1 > /dev/null
+
+ # let the client reconnect
+ client_reconnect
+ EVICT=$($LCTL get_param mdc.$FSNAME-MDT*.state |
+ awk -F"[ [,]" '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+
+ [ -z "$EVICT" ] || [[ $EVICT -le $BEFORE ]] || error "eviction happened"
+}
+run_test 113 "ldlm enqueue dropped reply should not cause deadlocks"
+
+complete $SECONDS