2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
7 # bug number: LU-2012 10124
8 ALWAYS_EXCEPT="14b 15c $REPLAY_DUAL_EXCEPT"
11 PTLDEBUG=${PTLDEBUG:--1}
12 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
14 CLEANUP=${CLEANUP:-""}
15 MOUNT_2=${MOUNT_2:-"yes"}
16 export MULTIOP=${MULTIOP:-multiop}
17 . $LUSTRE/tests/test-framework.sh
20 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
23 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
25 [ "$SLOW" = "no" ] && EXCEPT_SLOW="21b"
29 check_and_setup_lustre
30 MOUNTED=$(mounted_lustre_filesystems)
31 if ! $(echo $MOUNTED' ' | grep -w -q $MOUNT2' '); then
32 zconf_mount $HOSTNAME $MOUNT2
37 rm -rf $DIR/[df][0-9]*
39 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
41 # if there is no CLIENT1 defined, some tests can be ran on localhost
42 CLIENT1=${CLIENT1:-$HOSTNAME}
43 # if CLIENT2 doesn't exist then use CLIENT1 instead
44 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
46 # Exception is the test which need two separate nodes
47 CLIENT2=${CLIENT2:-$CLIENT1}
49 # LU-482 Avert LVM and VM inability to flush caches in pre .33 kernels
50 if [ $LINUX_VERSION_CODE -lt $(version_code 2.6.33) ]; then
52 do_facet $SINGLEMDS "sync; sleep 10; sync; sleep 10; sync"
55 LU482_FAILED=$(mktemp -u $TMP/$TESTSUITE.lu482.XXXXXX)
57 echo "Check file is LU482_FAILED=$LU482_FAILED"
58 touch $MOUNT2/$tfile-A # force sync FLD/SEQ update before barrier
59 replay_barrier $SINGLEMDS
60 #define OBD_FAIL_PTLRPC_FINISH_REPLAY | OBD_FAIL_ONCE
62 createmany -o $MOUNT1/$tfile- 50
63 $LCTL set_param fail_loc=0x80000514
64 facet_failover $SINGLEMDS
65 [ -f "$LU482_FAILED" ] && skip "LU-482 failure" && return 0
69 zconf_mount `hostname` $MOUNT2 || error "mount2 fais"
70 unlinkmany $MOUNT1/$tfile- 50 || return 2
71 rm $MOUNT2/$tfile || return 3
72 rm $MOUNT2/$tfile-A || return 4
74 run_test 0a "expired recovery with lost client"
76 if [ -f "$LU482_FAILED" ]; then
77 log "Found check file $LU482_FAILED, aborting test script"
78 rm -vf "$LU482_FAILED"
80 do_nodes $CLIENTS umount -f $MOUNT2 || true
81 do_nodes $CLIENTS umount -f $MOUNT || true
82 # copied from stopall, but avoid the MDS recovery
83 for num in `seq $OSTCOUNT`; do
85 rm -f $TMP/ost${num}active
87 if ! combined_mgs_mds ; then
95 replay_barrier $SINGLEMDS
97 touch $MOUNT1/$tfile-2
99 facet_failover $SINGLEMDS
101 zconf_mount `hostname` $MOUNT1 || error "mount1 fais"
102 zconf_mount `hostname` $MOUNT2 || error "mount2 fais"
103 checkstat $MOUNT1/$tfile-2 && return 1
104 checkstat $MOUNT2/$tfile && return 2
107 run_test 0b "lost client during waiting for next transno"
111 replay_barrier $SINGLEMDS
115 checkstat $MOUNT2/a || return 1
116 checkstat $MOUNT1/b || return 2
117 rm $MOUNT2/a $MOUNT1/b
118 checkstat $MOUNT1/a && return 3
119 checkstat $MOUNT2/b && return 4
123 run_test 1 "|X| simple create"
127 replay_barrier $SINGLEMDS
131 checkstat $MOUNT2/adir || return 1
133 checkstat $MOUNT2/adir && return 2
136 run_test 2 "|X| mkdir adir"
139 replay_barrier $SINGLEMDS
141 mkdir $MOUNT2/adir/bdir
144 checkstat $MOUNT2/adir || return 1
145 checkstat $MOUNT1/adir/bdir || return 2
146 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
147 checkstat $MOUNT1/adir && return 3
148 checkstat $MOUNT2/adir/bdir && return 4
151 run_test 3 "|X| mkdir adir, mkdir adir/bdir "
155 replay_barrier $SINGLEMDS
156 mkdir $MOUNT1/adir && return 1
157 mkdir $MOUNT2/adir/bdir
160 checkstat $MOUNT2/adir || return 2
161 checkstat $MOUNT1/adir/bdir || return 3
163 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
164 checkstat $MOUNT1/adir && return 4
165 checkstat $MOUNT2/adir/bdir && return 5
168 run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir "
172 # multiclient version of replay_single.sh/test_8
174 multiop_bg_pause $MOUNT2/a o_tSc || return 1
177 replay_barrier $SINGLEMDS
179 wait $pid || return 1
182 [ -e $MOUNT2/a ] && return 2
185 run_test 5 "open, unlink |X| close"
190 multiop_bg_pause $MOUNT2/a o_c || return 1
192 multiop_bg_pause $MOUNT1/a o_c || return 1
195 replay_barrier $SINGLEMDS
197 wait $pid1 || return 1
201 wait $pid2 || return 1
202 [ -e $MOUNT2/a ] && return 2
205 run_test 6 "open1, open2, unlink |X| close1 [fail $SINGLEMDS] close2"
208 replay_barrier $SINGLEMDS
209 drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1
211 checkstat $MOUNT2/$tfile || return 2
212 rm $MOUNT1/$tfile || return 3
216 run_test 8 "replay of resent request"
219 replay_barrier $SINGLEMDS
220 mcreate $MOUNT1/$tfile-1
221 mcreate $MOUNT2/$tfile-2
222 # drop first reint reply
223 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119
225 do_facet $SINGLEMDS lctl set_param fail_loc=0
227 rm $MOUNT1/$tfile-[1,2] || return 1
231 run_test 9 "resending a replayed create"
234 mcreate $MOUNT1/$tfile-1
235 replay_barrier $SINGLEMDS
236 munlink $MOUNT1/$tfile-1
237 mcreate $MOUNT2/$tfile-2
238 # drop first reint reply
239 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119
241 do_facet $SINGLEMDS lctl set_param fail_loc=0
243 checkstat $MOUNT1/$tfile-1 && return 1
244 checkstat $MOUNT1/$tfile-2 || return 2
249 run_test 10 "resending a replayed unlink"
252 replay_barrier $SINGLEMDS
253 mcreate $MOUNT1/$tfile-1
254 mcreate $MOUNT2/$tfile-2
255 mcreate $MOUNT1/$tfile-3
256 mcreate $MOUNT2/$tfile-4
257 mcreate $MOUNT1/$tfile-5
258 # drop all reint replies for a while
259 do_facet $SINGLEMDS lctl set_param fail_loc=0x0119
260 # note that with this fail_loc set, facet_failover df will fail
261 facet_failover $SINGLEMDS
262 #sleep for while, let both clients reconnect and timeout
263 sleep $((TIMEOUT * 2))
264 do_facet $SINGLEMDS lctl set_param fail_loc=0
266 rm $MOUNT1/$tfile-[1-5] || return 1
270 run_test 11 "both clients timeout during replay"
273 replay_barrier $SINGLEMDS
275 multiop_bg_pause $DIR/$tfile mo_c || return 1
278 #define OBD_FAIL_LDLM_ENQUEUE_NET 0x302
279 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000302
280 facet_failover $SINGLEMDS
281 do_facet $SINGLEMDS lctl set_param fail_loc=0
282 clients_up || return 1
285 kill -USR1 $MULTIPID || return 3
286 wait $MULTIPID || return 4
287 $CHECKSTAT -t file $DIR/$tfile || return 2
292 run_test 12 "open resend timeout"
295 multiop_bg_pause $DIR/$tfile mo_c || return 1
298 replay_barrier $SINGLEMDS
300 kill -USR1 $MULTIPID || return 3
301 wait $MULTIPID || return 4
304 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000115
305 facet_failover $SINGLEMDS
306 do_facet $SINGLEMDS lctl set_param fail_loc=0
307 clients_up || return 1
310 $CHECKSTAT -t file $DIR/$tfile || return 2
315 run_test 13 "close resend timeout"
317 # test 14a removed after 18143 because it shouldn't fail anymore and do the same
322 wait_delete_completed
324 local BEFOREUSED=$(df -P $DIR | tail -1 | awk '{ print $3 }')
326 mkdir -p $MOUNT1/$tdir
327 $SETSTRIPE -i 0 $MOUNT1/$tdir
328 replay_barrier $SINGLEMDS
329 createmany -o $MOUNT1/$tdir/$tfile- 5
331 $SETSTRIPE -i 0 $MOUNT2/$tfile-2
332 dd if=/dev/zero of=$MOUNT2/$tfile-2 bs=1M count=5
333 createmany -o $MOUNT1/$tdir/$tfile-3- 5
337 wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
339 # first set of files should have been replayed
340 unlinkmany $MOUNT1/$tdir/$tfile- 5 || error "first unlinks failed"
341 unlinkmany $MOUNT1/$tdir/$tfile-3- 5 || error "second unlinks failed"
343 zconf_mount $HOSTNAME $MOUNT2 || error "mount $MOUNT2 failed"
344 [ -f $MOUNT2/$tfile-2 ] && error "$MOUNT2/$tfile-2 exists!"
346 wait_mds_ost_sync || error "wait_mds_ost_sync failed"
347 wait_delete_completed || error "wait_delete_complete failed"
349 local AFTERUSED=$(df -P $DIR | tail -1 | awk '{ print $3 }')
350 log "before $BEFOREUSED, after $AFTERUSED"
351 # leave some margin for some files/dirs to be modified (OI, llog, etc)
352 [ $AFTERUSED -gt $((BEFOREUSED + 128)) ] &&
353 error "after $AFTERUSED > before $BEFOREUSED" || true
355 run_test 14b "delete ost orphans if gap occured in objids due to VBR"
357 test_15a() { # was test_15
358 replay_barrier $SINGLEMDS
359 createmany -o $MOUNT1/$tfile- 25
360 createmany -o $MOUNT2/$tfile-2- 1
365 unlinkmany $MOUNT1/$tfile- 25 || return 2
366 [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists"
368 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
371 run_test 15a "timeout waiting for lost client during replay, 1 client completes"
374 replay_barrier $SINGLEMDS
375 for ((i = 0; i < 2000; i++)); do
376 echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed"
382 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
385 run_test 15c "remove multiple OST orphans"
388 replay_barrier $SINGLEMDS
389 createmany -o $MOUNT1/$tfile- 25
390 createmany -o $MOUNT2/$tfile-2- 1
393 facet_failover $SINGLEMDS
397 unlinkmany $MOUNT1/$tfile- 25 || return 2
399 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
403 run_test 16 "fail MDS during recovery (3571)"
406 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
408 createmany -o $MOUNT1/$tfile- 25
409 createmany -o $MOUNT2/$tfile-2- 1
411 # Make sure the disconnect is lost
419 unlinkmany $MOUNT1/$tfile- 25 || return 2
421 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
425 run_test 17 "fail OST during recovery (3571)"
427 # cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it
430 test_18() { # bug 3822 - evicting client with enqueued lock
432 mkdir -p $MOUNT1/$tdir
433 touch $MOUNT1/$tdir/f0
434 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED 0x30b
435 statmany -s $MOUNT1/$tdir/f 1 500 &
438 do_facet $SINGLEMDS lctl set_param fail_loc=0x8000030b # hold enqueue
440 #define OBD_FAIL_LDLM_BL_CALLBACK_NET 0x305
441 do_facet client lctl set_param fail_loc=0x80000305 # drop cb, evict
443 usleep 500 # wait to ensure first client is one that will be evicted
444 openfile -f O_RDONLY $MOUNT2/$tdir/f0
446 dmesg | grep "entering recovery in server" && \
447 error "client not evicted" || true
448 do_facet client "lctl set_param fail_loc=0"
449 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
451 run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)"
453 test_19() { # Bug 10991 - resend of open request does not fail assertion.
454 replay_barrier $SINGLEMDS
455 drop_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1
457 checkstat $DIR2/${tfile}0 || return 2
458 rm $DIR/${tfile}0 || return 3
462 run_test 19 "resend of open request"
466 replay_barrier $SINGLEMDS
472 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
473 TIER1=$((`date +%s` - BEFORE))
475 replay_barrier $SINGLEMDS
481 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
482 TIER2=$((`date +%s` - BEFORE))
483 [ $TIER2 -ge $((TIER1 * 2)) ] && \
484 error "recovery time is growing $TIER2 > $TIER1"
487 run_test 20 "recovery time is not increasing"
489 # commit on sharing tests
491 local param_file=$TMP/$tfile-params
493 save_lustre_params $(facet_active_host $SINGLEMDS) "mdt.*.commit_on_sharing" > $param_file
494 do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=1
495 touch $MOUNT1/$tfile-1
496 mv $MOUNT2/$tfile-1 $MOUNT2/$tfile-2
497 mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3
498 replay_barrier_nosync $SINGLEMDS
501 facet_failover $SINGLEMDS
503 # all renames are replayed
504 unlink $MOUNT1/$tfile-3 || return 2
506 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
508 do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=0
509 rm -rf $MOUNT1/$tfile-*
510 restore_lustre_params < $param_file
514 run_test 21a "commit on sharing"
518 do_node $CLIENT1 rm -f $MOUNT1/$tfile-*
521 do_node $CLIENT1 touch $MOUNT1/$tfile-1
522 do_node $CLIENT2 mv $MOUNT1/$tfile-1 $MOUNT1/$tfile-2
523 do_node $CLIENT1 mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3
525 replay_barrier_nosync $mds
526 shutdown_client $CLIENT2 $MOUNT1
530 # were renames replayed?
532 echo UNLINK $MOUNT1/$tfile-3
533 do_node $CLIENT1 unlink $MOUNT1/$tfile-3 || \
534 { echo "unlink $tfile-3 fail!" && rc=1; }
537 zconf_mount_clients $CLIENT2 $MOUNT1 || error "mount $CLIENT2 $MOUNT1 fail"
543 [ -z "$CLIENTS" ] && skip "Need two or more clients." && return
544 [ $CLIENTCOUNT -lt 2 ] && \
545 { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
547 if [ "$FAILURE_MODE" = "HARD" ] && mixed_mdt_devs; then
548 skip "Several mdt services on one mds node are used with FAILURE_MODE=$FAILURE_MODE. "
553 zconf_umount_clients $CLIENTS $MOUNT2
554 zconf_mount_clients $CLIENTS $MOUNT1
556 local param_file=$TMP/$tfile-params
558 local num=$(get_mds_dir $MOUNT1)
560 save_lustre_params $(facet_active_host mds$num) "mdt.*.commit_on_sharing" > $param_file
564 do_facet mds$num lctl set_param mdt.*.commit_on_sharing=$COS
566 test_21b_sub mds$num || error "Not all renames are replayed. COS=$COS"
568 # COS disabled (should fail)
570 do_facet mds$num lctl set_param mdt.*.commit_on_sharing=$COS
572 # there is still a window when transactions may be written to disk before
573 # the mds device is set R/O. To avoid such a rare test failure, the check
574 # is repeated several times.
577 test_21b_sub mds$num || break;
578 let n_attempts=n_attempts+1
579 [ $n_attempts -gt 3 ] &&
580 error "The test cannot check whether COS works or not: all renames are replied w/o COS"
582 zconf_mount_clients $CLIENTS $MOUNT2
583 restore_lustre_params < $param_file
587 run_test 21b "commit on sharing, two clients"
590 checkstat $MOUNT1/$remote_dir || return 1
591 checkstat $MOUNT1/$remote_dir/dir || return 2
592 checkstat $MOUNT1/$remote_dir/$tfile-1 || return 3
593 checkstat $MOUNT1/$remote_dir/dir/$tfile-1 || return 4
597 create_remote_dir_files_22() {
598 do_node $CLIENT2 mkdir ${MOUNT2}/$remote_dir/dir || return 1
599 do_node $CLIENT1 createmany -o $MOUNT1/$remote_dir/dir/$tfile- 2 ||
601 do_node $CLIENT2 createmany -o $MOUNT2/$remote_dir/$tfile- 2 ||
607 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
608 ([ $FAILURE_MODE == "HARD" ] &&
609 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
610 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
614 local remote_dir=${tdir}/remote_dir
616 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
618 # OBD_FAIL_MDS_REINT_NET_REP 0x119
619 do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
620 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
622 do_facet mds${MDTIDX} lctl set_param fail_loc=0
625 wait $CLIENT_PID || error "lfs mkdir failed"
627 replay_barrier mds${MDTIDX}
628 create_remote_dir_files_22 || error "Remote creation failed $?"
631 checkstat_22 || error "check stat failed $?"
633 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
636 run_test 22a "c1 lfs mkdir -i 1 dir1, M0 drop reply & fail, c2 mkdir dir1/dir"
639 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
641 local remote_dir=$tdir/remote_dir
643 # OBD_FAIL_MDS_REINT_NET_REP 0x119
644 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
646 do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
647 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
649 do_facet mds${MDTIDX} lctl set_param fail_loc=0
651 fail mds${MDTIDX},mds$((MDTIDX + 1))
652 wait $CLIENT_PID || error "lfs mkdir failed"
654 replay_barrier mds$MDTIDX
655 create_remote_dir_files_22 || error "Remote creation failed $?"
658 checkstat_22 || error "check stat failed $?"
660 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
663 run_test 22b "c1 lfs mkdir -i 1 d1, M0 drop reply & fail M0/M1, c2 mkdir d1/dir"
666 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
667 ([ $FAILURE_MODE == "HARD" ] &&
668 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
669 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
672 local remote_dir=${tdir}/remote_dir
674 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
676 # OBD_FAIL_MDS_DROP_OBJ_UPDATE 0x188
677 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x188
678 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
680 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
682 fail mds$((MDTIDX+1))
683 wait $CLIENT_PID || error "lfs mkdir failed"
685 replay_barrier mds$MDTIDX
686 create_remote_dir_files_22 || error "Remote creation failed $?"
689 checkstat_22 || error "check stat failed $?"
691 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
694 run_test 22c "c1 lfs mkdir -i 1 d1, M1 drop update & fail M1, c2 mkdir d1/dir"
697 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
699 local remote_dir=${tdir}/remote_dir
701 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
703 # OBD_FAIL_MDS_DROP_OBJ_UPDATE 0x188
704 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x188
705 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
707 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
709 fail mds${MDTIDX},mds$((MDTIDX + 1))
710 wait $CLIENT_PID || error "lfs mkdir failed"
712 replay_barrier mds$MDTIDX
713 create_remote_dir_files_22 || error "Remote creation failed $?"
716 checkstat_22 || error "check stat failed $?"
718 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
721 run_test 22d "c1 lfs mkdir -i 1 d1, M1 drop update & fail M0/M1,c2 mkdir d1/dir"
724 checkstat $MOUNT1/$remote_dir || return 1
725 checkstat $MOUNT1/$remote_dir/$tfile-1 || return 2
729 create_remote_dir_files_23() {
730 do_node $CLIENT2 mkdir ${MOUNT2}/$remote_dir || return 1
731 do_node $CLIENT2 createmany -o $MOUNT2/$remote_dir/$tfile- 2 || return 2
736 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
737 ([ $FAILURE_MODE == "HARD" ] &&
738 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
739 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
742 local remote_dir=$tdir/remote_dir
744 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
745 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
746 error "lfs mkdir failed"
747 # OBD_FAIL_MDS_REINT_NET_REP 0x119
748 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
749 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
751 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
753 fail mds$((MDTIDX + 1))
754 wait $CLIENT_PID || error "rmdir remote dir failed"
756 replay_barrier mds${MDTIDX}
757 create_remote_dir_files_23 || error "Remote creation failed $?"
760 checkstat_23 || error "check stat failed $?"
762 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
765 run_test 23a "c1 rmdir d1, M1 drop reply and fail, client2 mkdir d1"
768 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
770 local remote_dir=$tdir/remote_dir
772 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
773 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
774 error "lfs mkdir failed"
776 # OBD_FAIL_MDS_REINT_NET_REP 0x119
777 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
778 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
780 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
782 fail mds${MDTIDX},mds$((MDTIDX + 1))
783 wait $CLIENT_PID || error "rmdir remote dir failed"
785 replay_barrier mds${MDTIDX}
786 create_remote_dir_files_23 || error "Remote creation failed $?"
789 checkstat_23 || error "check stat failed $?"
791 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
794 run_test 23b "c1 rmdir d1, M1 drop reply and fail M0/M1, c2 mkdir d1"
797 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
799 ([ $FAILURE_MODE == "HARD" ] &&
800 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
801 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
804 local remote_dir=$tdir/remote_dir
806 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
807 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
808 error "lfs mkdir failed"
810 # OBD_FAIL_MDS_DROP_OBJ_UPDATE 0x188
811 do_facet mds${MDTIDX} lctl set_param fail_loc=0x188
812 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
814 do_facet mds${MDTIDX} lctl set_param fail_loc=0
817 wait $CLIENT_PID || error "rmdir remote dir failed"
819 replay_barrier mds${MDTIDX}
820 create_remote_dir_files_23 || error "Remote creation failed $?"
823 checkstat_23 || error "check stat failed $?"
825 rm -rf $MOUNT1/$tdir || return 6
828 run_test 23c "c1 rmdir d1, M0 drop update reply and fail M0, c2 mkdir d1"
831 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
833 local remote_dir=$tdir/remote_dir
835 do_node $CLIENT1 mkdir -p $MOUNT1/${tdir}
836 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
837 error "lfs mkdir failed"
839 # OBD_FAIL_MDS_DROP_OBJ_UPDATE 0x188
840 do_facet mds${MDTIDX} lctl set_param fail_loc=0x188
841 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
843 do_facet mds${MDTIDX} lctl set_param fail_loc=0
845 fail mds${MDTIDX},mds$((MDTIDX + 1))
846 wait $CLIENT_PID || error "rmdir remote dir failed"
848 replay_barrier mds${MDTIDX}
849 create_remote_dir_files_23 || error "Remote creation failed $?"
852 checkstat_23 || error "check stat failed $?"
854 rm -rf $MOUNT1/$tdir || return 6
857 run_test 23d "c1 rmdir d1, M0 drop update reply and fail M0/M1, c2 mkdir d1"
859 # end commit on sharing tests
862 SLEEP=$((`date +%s` - $NOW))
863 [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
864 [ "$MOUNTED2" = yes ] && zconf_umount $HOSTNAME $MOUNT2 || true
865 check_and_cleanup_lustre