5 PTLDEBUG=${PTLDEBUG:--1}
6 MOUNT_2=${MOUNT_2:-"yes"}
7 LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"}
9 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
10 . $LUSTRE/tests/test-framework.sh
14 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
16 ALWAYS_EXCEPT="$REPLAY_DUAL_EXCEPT "
17 # bug number for skipped test: LU-2012 LU-8333
18 ALWAYS_EXCEPT+=" 14b 21b"
19 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
21 [[ "$mds1_FSTYPE" == zfs ]] &&
22 # bug number for skipped test: LU-2230
23 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 21b"
26 [ "$SLOW" = "no" ] && EXCEPT_SLOW="21b "
29 check_and_setup_lustre
31 MOUNTED=$(mounted_lustre_filesystems)
32 if ! $(echo $MOUNTED' ' | grep -w -q $MOUNT2' '); then
33 zconf_mount $HOSTNAME $MOUNT2
38 rm -rf $DIR/[df][0-9]*
40 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
42 # if there is no CLIENT1 defined, some tests can be ran on localhost
43 CLIENT1=${CLIENT1:-$HOSTNAME}
44 # if CLIENT2 doesn't exist then use CLIENT1 instead
45 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
47 # Exception is the test which need two separate nodes
48 CLIENT2=${CLIENT2:-$CLIENT1}
52 LU482_FAILED=$(mktemp -u $TMP/$TESTSUITE.lu482.XXXXXX)
54 echo "Check file is LU482_FAILED=$LU482_FAILED"
55 touch $MOUNT2/$tfile-A # force sync FLD/SEQ update before barrier
56 replay_barrier $SINGLEMDS
57 #define OBD_FAIL_PTLRPC_FINISH_REPLAY | CFS_FAIL_ONCE
59 createmany -o $MOUNT1/$tfile- 50
60 $LCTL set_param fail_loc=0x80000514
61 facet_failover $SINGLEMDS
62 [ -f "$LU482_FAILED" ] && skip "LU-482 failure" && return 0
63 client_up || (sleep 10; client_up) || (sleep 10; client_up) ||
64 error "reconnect failed"
66 client_up || (sleep 10; client_up) || (sleep 10; client_up) ||
67 error "reconnect failed"
68 zconf_mount `hostname` $MOUNT2 || error "mount2 failed"
69 unlinkmany $MOUNT1/$tfile- 50 || errot "unlinkmany failed"
70 rm $MOUNT2/$tfile || error "rm $MOUNT2/$tfile failed"
71 rm $MOUNT2/$tfile-A || error "rm $MOUNT2/$tfile-A failed"
73 run_test 0a "expired recovery with lost client"
75 if [ -f "$LU482_FAILED" ]; then
76 log "Found check file $LU482_FAILED, aborting test script"
77 rm -vf "$LU482_FAILED"
78 complete_test $SECONDS
79 do_nodes $CLIENTS umount -f $MOUNT2 || true
80 do_nodes $CLIENTS umount -f $MOUNT || true
81 # copied from stopall, but avoid the MDS recovery
82 for num in `seq $OSTCOUNT`; do
84 rm -f $TMP/ost${num}active
86 if ! combined_mgs_mds ; then
94 replay_barrier $SINGLEMDS
96 touch $MOUNT1/$tfile-2
98 facet_failover $SINGLEMDS
100 zconf_mount `hostname` $MOUNT1 || error "mount1 fais"
101 zconf_mount `hostname` $MOUNT2 || error "mount2 fais"
102 # it is uncertain if file-2 exists or not, remove it if it does
103 checkstat $MOUNT1/$tfile-2 && rm $MOUNT1/$tfile-2
104 checkstat $MOUNT2/$tfile && return 2
107 run_test 0b "lost client during waiting for next transno"
111 replay_barrier $SINGLEMDS
115 checkstat $MOUNT2/a || return 1
116 checkstat $MOUNT1/b || return 2
117 rm $MOUNT2/a $MOUNT1/b
118 checkstat $MOUNT1/a && return 3
119 checkstat $MOUNT2/b && return 4
123 run_test 1 "|X| simple create"
127 replay_barrier $SINGLEMDS
131 checkstat $MOUNT2/adir || return 1
133 checkstat $MOUNT2/adir && return 2
136 run_test 2 "|X| mkdir adir"
139 replay_barrier $SINGLEMDS
141 mkdir $MOUNT2/adir/bdir
144 checkstat $MOUNT2/adir || return 1
145 checkstat $MOUNT1/adir/bdir || return 2
146 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
147 checkstat $MOUNT1/adir && return 3
148 checkstat $MOUNT2/adir/bdir && return 4
151 run_test 3 "|X| mkdir adir, mkdir adir/bdir "
155 replay_barrier $SINGLEMDS
156 mkdir $MOUNT1/adir && return 1
157 mkdir $MOUNT2/adir/bdir
160 checkstat $MOUNT2/adir || return 2
161 checkstat $MOUNT1/adir/bdir || return 3
163 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
164 checkstat $MOUNT1/adir && return 4
165 checkstat $MOUNT2/adir/bdir && return 5
168 run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir "
172 # multiclient version of replay_single.sh/test_8
174 multiop_bg_pause $MOUNT2/a o_tSc || return 1
177 replay_barrier $SINGLEMDS
179 wait $pid || return 1
182 [ -e $MOUNT2/a ] && return 2
185 run_test 5 "open, unlink |X| close"
190 multiop_bg_pause $MOUNT2/a o_c || return 1
192 multiop_bg_pause $MOUNT1/a o_c || return 1
195 replay_barrier $SINGLEMDS
197 wait $pid1 || return 1
201 wait $pid2 || return 1
202 [ -e $MOUNT2/a ] && return 2
205 run_test 6 "open1, open2, unlink |X| close1 [fail $SINGLEMDS] close2"
208 replay_barrier $SINGLEMDS
209 drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1
211 checkstat $MOUNT2/$tfile || return 2
212 rm $MOUNT1/$tfile || return 3
216 run_test 8 "replay of resent request"
219 replay_barrier $SINGLEMDS
220 mcreate $MOUNT1/$tfile-1
221 mcreate $MOUNT2/$tfile-2
222 # drop first reint reply
223 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119
225 do_facet $SINGLEMDS lctl set_param fail_loc=0
227 rm $MOUNT1/$tfile-[1,2] || return 1
231 run_test 9 "resending a replayed create"
234 mcreate $MOUNT1/$tfile-1
235 replay_barrier $SINGLEMDS
236 munlink $MOUNT1/$tfile-1
237 mcreate $MOUNT2/$tfile-2
238 # drop first reint reply
239 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119
241 do_facet $SINGLEMDS lctl set_param fail_loc=0
243 checkstat $MOUNT1/$tfile-1 && return 1
244 checkstat $MOUNT1/$tfile-2 || return 2
249 run_test 10 "resending a replayed unlink"
252 replay_barrier $SINGLEMDS
253 mcreate $DIR1/$tfile-1
254 mcreate $DIR2/$tfile-2
255 mcreate $DIR1/$tfile-3
256 mcreate $DIR2/$tfile-4
257 mcreate $DIR1/$tfile-5
258 # drop all reint replies for a while
259 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0119
260 # note that with this fail_loc set, facet_failover df will fail
261 facet_failover $SINGLEMDS
263 local clients=${CLIENTS:-$HOSTNAME}
264 wait_clients_import_state "$clients" $SINGLEMDS FULL
266 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
268 rm $DIR1/$tfile-[1-5] || return 1
272 run_test 11 "both clients timeout during replay"
275 replay_barrier $SINGLEMDS
277 multiop_bg_pause $DIR/$tfile mo_c || return 1
280 #define OBD_FAIL_LDLM_ENQUEUE_NET 0x302
281 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000302
282 facet_failover $SINGLEMDS
283 do_facet $SINGLEMDS lctl set_param fail_loc=0
284 clients_up || return 1
287 kill -USR1 $MULTIPID || return 3
288 wait $MULTIPID || return 4
289 $CHECKSTAT -t file $DIR/$tfile || return 2
294 run_test 12 "open resend timeout"
297 multiop_bg_pause $DIR/$tfile mo_c || return 1
300 replay_barrier $SINGLEMDS
302 kill -USR1 $MULTIPID || return 3
303 wait $MULTIPID || return 4
306 do_facet $SINGLEMDS lctl set_param fail_loc=0x80000115
307 facet_failover $SINGLEMDS
308 do_facet $SINGLEMDS lctl set_param fail_loc=0
309 clients_up || return 1
312 $CHECKSTAT -t file $DIR/$tfile || return 2
317 run_test 13 "close resend timeout"
319 # test 14a removed after 18143 because it shouldn't fail anymore and do the same
324 wait_delete_completed
326 local beforeused=$(df -P $DIR | tail -1 | awk '{ print $3 }')
328 mkdir -p $MOUNT1/$tdir
329 $LFS setstripe -i 0 $MOUNT1/$tdir
330 replay_barrier $SINGLEMDS
331 createmany -o $MOUNT1/$tdir/$tfile- 5
333 $LFS setstripe -i 0 $MOUNT2/$tfile-2
334 dd if=/dev/zero of=$MOUNT2/$tfile-2 bs=1M count=5
335 createmany -o $MOUNT1/$tdir/$tfile-3- 5
339 wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
341 # first set of files should have been replayed
342 unlinkmany $MOUNT1/$tdir/$tfile- 5 || error "first unlinks failed"
343 unlinkmany $MOUNT1/$tdir/$tfile-3- 5 || error "second unlinks failed"
345 zconf_mount $HOSTNAME $MOUNT2 || error "mount $MOUNT2 failed"
346 [ -f $MOUNT2/$tfile-2 ] && error "$MOUNT2/$tfile-2 exists!"
348 wait_mds_ost_sync || error "wait_mds_ost_sync failed"
349 wait_delete_completed || error "wait_delete_complete failed"
351 local afterused=$(df -P $DIR | tail -1 | awk '{ print $3 }')
352 log "before $beforeused, after $afterused"
353 # leave some margin for some files/dirs to be modified (OI, llog, etc)
354 [ $afterused -le $((beforeused + $(fs_log_size))) ] ||
355 error "after $afterused > before $beforeused"
357 run_test 14b "delete ost orphans if gap occured in objids due to VBR"
359 test_15a() { # was test_15
360 replay_barrier $SINGLEMDS
361 createmany -o $MOUNT1/$tfile- 25
362 createmany -o $MOUNT2/$tfile-2- 1
367 unlinkmany $MOUNT1/$tfile- 25 || return 2
368 [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists"
370 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
373 run_test 15a "timeout waiting for lost client during replay, 1 client completes"
376 replay_barrier $SINGLEMDS
377 for ((i = 0; i < 2000; i++)); do
378 echo "data" > "$MOUNT2/${tfile}-$i" ||
379 error "create ${tfile}-$i failed"
385 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
388 run_test 15c "remove multiple OST orphans"
391 replay_barrier $SINGLEMDS
392 createmany -o $MOUNT1/$tfile- 25
393 createmany -o $MOUNT2/$tfile-2- 1
396 facet_failover $SINGLEMDS
400 unlinkmany $MOUNT1/$tfile- 25 || return 2
402 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
405 run_test 16 "fail MDS during recovery (3571)"
408 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
410 createmany -o $MOUNT1/$tfile- 25
411 createmany -o $MOUNT2/$tfile-2- 1
413 # Make sure the disconnect is lost
421 unlinkmany $MOUNT1/$tfile- 25 || return 2
423 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
427 run_test 17 "fail OST during recovery (3571)"
429 # cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it
432 test_18() { # bug 3822 - evicting client with enqueued lock
434 local DLMTRACE=$(do_facet $SINGLEMDS lctl get_param debug)
435 do_facet $SINGLEMDS lctl set_param debug=+dlmtrace
436 mkdir -p $MOUNT1/$tdir || error "mkdir $MOUNT1/$tdir failed"
437 touch $MOUNT1/$tdir/${tfile}0 || error "touch file failed"
438 statmany -s $MOUNT1/$tdir/$tfile 1 500 &
441 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED 0x30b
442 do_facet $SINGLEMDS lctl set_param fail_loc=0x8000030b # hold enqueue
444 #define OBD_FAIL_LDLM_BL_CALLBACK_NET 0x305
445 do_facet client lctl set_param ldlm.namespaces.*.early_lock_cancel=0
446 do_facet client lctl set_param fail_loc=0x80000305 # drop cb, evict
448 sleep 0.1 # wait to ensure first client is one that will be evicted
449 openfile -f O_RDONLY $MOUNT2/$tdir/$tfile
451 do_facet client lctl set_param ldlm.namespaces.*.early_lock_cancel=1
452 do_facet $SINGLEMDS lctl debug_kernel |
453 grep "not entering recovery" && error "client not evicted"
454 do_facet client "lctl set_param fail_loc=0"
455 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
457 run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)"
459 test_19() { # Bug 10991 - resend of open request does not fail assertion.
460 replay_barrier $SINGLEMDS
461 drop_mdt_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1
463 checkstat $DIR2/${tfile}0 || return 2
464 rm $DIR/${tfile}0 || return 3
468 run_test 19 "resend of open request"
471 local before=$SECONDS
472 replay_barrier $SINGLEMDS
478 zconf_mount $HOSTNAME $DIR2 || error "mount $DIR2 fail"
479 local tier1=$((SECONDS - before))
482 replay_barrier $SINGLEMDS
488 zconf_mount $HOSTNAME $DIR2 || error "mount $DIR2 fail"
489 local tier2=$((SECONDS - before))
491 # timeout is more than 1.5x original timeout
492 ((tier2 < tier1 * 6 / 4)) ||
493 error "recovery time $tier2 >= 1.5x original time $tier1"
495 run_test 20 "recovery time is not increasing"
497 # commit on sharing tests
499 local param_file=$TMP/$tfile-params
501 save_lustre_params $SINGLEMDS "mdt.*.commit_on_sharing" > $param_file
502 do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=1
503 touch $MOUNT1/$tfile-1
504 mv $MOUNT2/$tfile-1 $MOUNT2/$tfile-2
505 mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3
506 replay_barrier_nosync $SINGLEMDS
509 facet_failover $SINGLEMDS
511 # all renames are replayed
512 unlink $MOUNT1/$tfile-3 || return 2
514 zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail"
516 do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=0
517 rm -rf $MOUNT1/$tfile-*
518 restore_lustre_params < $param_file
522 run_test 21a "commit on sharing"
526 do_node $CLIENT1 rm -f $MOUNT1/$tfile-*
529 do_node $CLIENT1 touch $MOUNT1/$tfile-1
530 do_node $CLIENT2 mv $MOUNT1/$tfile-1 $MOUNT1/$tfile-2
531 do_node $CLIENT1 mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3
533 replay_barrier_nosync $mds
534 shutdown_client $CLIENT2 $MOUNT1
538 # were renames replayed?
540 echo UNLINK $MOUNT1/$tfile-3
541 do_node $CLIENT1 unlink $MOUNT1/$tfile-3 ||
542 { echo "unlink $tfile-3 fail!" && rc=1; }
545 zconf_mount_clients $CLIENT2 $MOUNT1 ||
546 error "mount $CLIENT2 $MOUNT1 fail"
552 [ $CLIENTCOUNT -lt 2 ] &&
553 { skip "Need 2+ clients, have $CLIENTCOUNT" && return; }
555 if [ "$FAILURE_MODE" = "HARD" ] && mixed_mdt_devs; then
556 skip "Several MDTs on one MDS with FAILURE_MODE=$FAILURE_MODE"
560 zconf_umount_clients $CLIENTS $MOUNT2
561 zconf_mount_clients $CLIENTS $MOUNT1
563 local param_file=$TMP/$tfile-params
565 local mdtidx=$($LFS getstripe -m $MOUNT1)
566 local facet=mds$((mdtidx + 1))
568 save_lustre_params $facet "mdt.*.commit_on_sharing" > $param_file
572 do_facet $facet lctl set_param mdt.*.commit_on_sharing=$COS
574 test_21b_sub $facet || error "Not all renames are replayed. COS=$COS"
576 # there is still a window when transactions may be written to disk
577 # before the mds device is set R/O. To avoid such a rare test failure,
578 # the check is repeated several times.
582 # COS disabled (should fail)
583 do_facet $facet lctl set_param mdt.*.commit_on_sharing=$COS
585 test_21b_sub $facet || break
586 n_attempts=$((n_attempts + 1))
587 [ $n_attempts -gt 3 ] &&
588 error "can't check if COS works: rename replied w/o COS"
590 zconf_mount_clients $CLIENTS $MOUNT2
591 restore_lustre_params < $param_file
595 run_test 21b "commit on sharing, two clients"
598 checkstat $MOUNT1/$remote_dir || return 1
599 checkstat $MOUNT1/$remote_dir/dir || return 2
600 checkstat $MOUNT1/$remote_dir/$tfile-1 || return 3
601 checkstat $MOUNT1/$remote_dir/dir/$tfile-1 || return 4
605 create_remote_dir_files_22() {
606 do_node $CLIENT2 mkdir ${MOUNT2}/$remote_dir/dir || return 1
607 do_node $CLIENT1 createmany -o $MOUNT1/$remote_dir/dir/$tfile- 2 ||
609 do_node $CLIENT2 createmany -o $MOUNT2/$remote_dir/$tfile- 2 ||
615 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
616 ([ $FAILURE_MODE == "HARD" ] &&
617 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
618 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
622 local remote_dir=$tdir/remote_dir
624 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
625 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
627 # OBD_FAIL_MDS_REINT_NET_REP 0x119
628 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
629 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
633 fail mds$((MDTIDX + 1))
634 wait $CLIENT_PID || error "lfs mkdir -i $MDTIDX failed"
636 replay_barrier mds$MDTIDX
637 create_remote_dir_files_22 || error "Remote creation failed $?"
640 checkstat_22 || error "check stat failed $?"
642 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
645 run_test 22a "c1 lfs mkdir -i 1 dir1, M1 drop reply & fail, c2 mkdir dir1/dir"
648 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
650 local remote_dir=$tdir/remote_dir
652 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
653 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
655 # OBD_FAIL_MDS_REINT_NET_REP 0x119
656 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
657 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
661 fail mds${MDTIDX},mds$((MDTIDX + 1))
662 wait $CLIENT_PID || error "lfs mkdir -i $MDTIDX failed"
664 replay_barrier mds$MDTIDX
665 create_remote_dir_files_22 || error "Remote creation failed $?"
668 checkstat_22 || error "check stat failed $?"
670 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
673 run_test 22b "c1 lfs mkdir -i 1 d1, M1 drop reply & fail M0/M1, c2 mkdir d1/dir"
676 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
677 ([ $FAILURE_MODE == "HARD" ] &&
678 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
679 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
682 local remote_dir=$tdir/remote_dir
684 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
685 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
687 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
688 do_facet mds$MDTIDX lctl set_param fail_loc=0x1701
689 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
692 do_facet mds$MDTIDX lctl set_param fail_loc=0
695 wait $CLIENT_PID || error "lfs mkdir -i $MDTIDX failed"
697 replay_barrier mds$MDTIDX
698 create_remote_dir_files_22 || error "Remote creation failed $?"
701 checkstat_22 || error "check stat failed $?"
703 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
706 run_test 22c "c1 lfs mkdir -i 1 d1, M1 drop update & fail M1, c2 mkdir d1/dir"
709 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
711 local remote_dir=$tdir/remote_dir
713 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
714 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
716 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
717 do_facet mds$MDTIDX lctl set_param fail_loc=0x1701
718 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir &
721 do_facet mds$MDTIDX lctl set_param fail_loc=0
723 fail mds${MDTIDX},mds$((MDTIDX + 1))
724 wait $CLIENT_PID || error "lfs mkdir -i $MDTIDX failed"
726 replay_barrier mds$MDTIDX
727 create_remote_dir_files_22 || error "Remote creation failed $?"
730 checkstat_22 || error "check stat failed $?"
732 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
735 run_test 22d "c1 lfs mkdir -i 1 d1, M1 drop update & fail M0/M1,c2 mkdir d1/dir"
738 checkstat $MOUNT1/$remote_dir || return 1
739 checkstat $MOUNT1/$remote_dir/$tfile-1 || return 2
743 create_remote_dir_files_23() {
744 do_node $CLIENT2 mkdir ${MOUNT2}/$remote_dir || return 1
745 do_node $CLIENT2 createmany -o $MOUNT2/$remote_dir/$tfile- 2 || return 2
750 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
751 ([ $FAILURE_MODE == "HARD" ] &&
752 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
753 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
756 local remote_dir=$tdir/remote_dir
758 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
759 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
760 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
761 error "lfs mkdir -i $MDTIDX failed"
762 # OBD_FAIL_MDS_REINT_NET_REP 0x119
763 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
764 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
767 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
769 fail mds$((MDTIDX + 1))
770 wait $CLIENT_PID || error "rmdir remote dir failed"
772 replay_barrier mds${MDTIDX}
773 create_remote_dir_files_23 || error "Remote creation failed $?"
776 checkstat_23 || error "check stat failed $?"
778 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
781 run_test 23a "c1 rmdir d1, M1 drop reply and fail, client2 mkdir d1"
784 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
786 local remote_dir=$tdir/remote_dir
788 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
789 error "lfs mkdir -i 0 $MOUNT/$tdir failed"
790 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
791 error "lfs mkdir -i $MDTIDX failed"
793 # OBD_FAIL_MDS_REINT_NET_REP 0x119
794 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
795 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
798 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
800 fail mds${MDTIDX},mds$((MDTIDX + 1))
801 wait $CLIENT_PID || error "rmdir remote dir failed"
803 replay_barrier mds${MDTIDX}
804 create_remote_dir_files_23 || error "Remote creation failed $?"
807 checkstat_23 || error "check stat failed $?"
809 rm -rf $MOUNT1/$tdir || error "rmdir remote_dir failed"
812 run_test 23b "c1 rmdir d1, M1 drop reply and fail M0/M1, c2 mkdir d1"
815 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
817 ([ $FAILURE_MODE == "HARD" ] &&
818 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
819 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
822 local remote_dir=$tdir/remote_dir
824 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
825 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
826 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
827 error "lfs mkdir -i $MDTIDX failed"
829 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
830 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
831 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
834 do_facet mds${MDTIDX} lctl set_param fail_loc=0
837 wait $CLIENT_PID || error "rmdir remote dir failed"
839 replay_barrier mds${MDTIDX}
840 create_remote_dir_files_23 || error "Remote creation failed $?"
843 checkstat_23 || error "check stat failed $?"
845 rm -rf $MOUNT1/$tdir || return 6
848 run_test 23c "c1 rmdir d1, M0 drop update reply and fail M0, c2 mkdir d1"
851 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
853 local remote_dir=$tdir/remote_dir
855 do_node $CLIENT1 $LFS mkdir -i 0 $MOUNT1/$tdir ||
856 error "lfs mkdir -i 0 $MOUNT1/$tdir failed"
857 do_node $CLIENT1 $LFS mkdir -i $MDTIDX $MOUNT1/$remote_dir ||
858 error "lfs mkdir -i $MDTIDX failed"
860 # let previous transactions to complete
861 # (distributed llog cancels, etc)
862 do_nodes $(comma_list $(mdts_nodes)) \
863 "$LCTL set_param -n osd*.*MDT*.force_sync=1"
866 # OBD_FAIL_UPDATE_OBJ_NET 0x1701
867 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
868 do_node $CLIENT1 rmdir $MOUNT1/$remote_dir &
871 do_facet mds${MDTIDX} lctl set_param fail_loc=0
873 fail mds${MDTIDX},mds$((MDTIDX + 1))
874 wait $CLIENT_PID || error "rmdir remote dir failed"
876 replay_barrier mds${MDTIDX}
877 create_remote_dir_files_23 || error "Remote creation failed $?"
880 checkstat_23 || error "check stat failed $?"
882 rm -rf $MOUNT1/$tdir || return 6
885 run_test 23d "c1 rmdir d1, M0 drop update reply and fail M0/M1, c2 mkdir d1"
888 [[ "$MDS1_VERSION" -gt $(version_code 2.5.2) ]] ||
889 skip "Need MDS version newer than 2.5.2"
892 stat $MOUNT/$tfile >&/dev/null
893 # OBD_FAIL_MDS_REINT_NET_REP
894 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x119
895 $TRUNCATE $MOUNT/$tfile 100 &
898 do_facet $SINGLEMDS lctl set_param fail_loc=0
899 # sync to release rep-ack lock quickly
900 do_nodes $(comma_list $(mdts_nodes)) \
901 "lctl set_param -n osd*.*MDT*.force_sync 1"
905 run_test 24 "reconstruct on non-existing object"
907 # end commit on sharing tests
912 $LFS setstripe -i 0 -c 1 $DIR/$tfile
914 # get lock for the 1st client
915 dd if=/dev/zero of=$DIR/$tfile count=1 >/dev/null ||
916 error "failed to write data"
918 # get waiting locks for the 2nd client
919 drop_ldlm_cancel "multiop $DIR2/$tfile Ow512" &
922 # failover, replay and resend replayed waiting locks
923 if [ "$OST1_VERSION" -ge $(version_code 2.6.90) ]; then
924 #define OBD_FAIL_LDLM_SRV_CP_AST 0x325
925 do_facet ost1 lctl set_param fail_loc=0x80000325
927 #define OBD_FAIL_OST_LDLM_REPLY_NET 0x213
928 do_facet ost1 lctl set_param fail_loc=0x80000213
933 # multiop does not finish because CP AST is skipped;
934 # it is ok to kill it in the test, because CP AST is already re-sent
935 # and it does not hung forever in real life
936 killall -r "(multiop)$"
939 run_test 25 "replay|resend"
947 for pid_26 in "${pids_26[@]}"; do
948 if [[ -n "$pid_26" ]]; then
949 kill -0 "$pid_26" && kill "$pid_26" && \
950 wait "$pid_26" || true
954 for dir_26 in "${dirs_26[@]}"; do
955 if [[ -n "$dir_26" && -d "$dir_26" ]]; then
962 local clients=${CLIENTS:-$HOSTNAME}
964 zconf_mount_clients $clients $MOUNT
967 [[ "$SLOW" == "no" ]] && duration=200
968 # set duration to 900 because it takes some time to boot node
969 [[ "$FAILURE_MODE" == HARD ]] && duration=900
971 local start_ts=$SECONDS
974 stack_trap cleanup_26
976 local tar_dir=$DIR/$tdir/run_tar
980 stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \
984 test_mkdir -p -c$MDSCOUNT $tar_dir || break
986 if (( MDSCOUNT >= 2 )); then
987 $LFS setdirstripe -D -c$MDSCOUNT $tar_dir ||
988 error "set default dirstripe failed"
992 tar -C / -cf - etc | tar -xf - &
997 wait $tar_pid || tar_rc=$?
999 if (( tar_rc > 0 && tar_rc <= 128 )); then
1000 error "tar failed with rc $tar_rc"
1003 cd $DIR/$tdir || break
1004 rm -rf $tar_dir || break
1010 echo "Started tar loop with pid $tar_26_pid"
1011 pids_26+=($tar_26_pid)
1013 local dbench_dir=$DIR2/$tdir/run_dbench
1015 dirs_26+=($dbench_dir)
1017 stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \
1021 test_mkdir -p -c$MDSCOUNT $dbench_dir || break
1023 if (( MDSCOUNT >= 2 )); then
1024 $LFS setdirstripe -D -c$MDSCOUNT $dbench_dir ||
1025 error "set default dirstripe failed"
1028 cd $dbench_dir || break
1029 bash rundbench 1 -D $dbench_dir -t 100 &
1034 wait $dbench_pid || dbench_rc=$?
1036 if (( dbench_rc > 0 && dbench_rc <= 128 )); then
1037 error "dbench failed with rc $dbench_rc"
1040 cd $DIR/$tdir || break
1041 rm -rf $dbench_dir || break
1045 local dbench_26_pid=$!
1047 echo "Started dbench loop with $dbench_26_pid"
1048 pids_26+=($dbench_26_pid)
1050 local num_failovers=0
1053 while (( (SECONDS - start_ts) < duration )); do
1054 kill -0 $tar_26_pid || error "tar $tar_26_pid missing"
1055 kill -0 $dbench_26_pid || error "dbench $dbench_26_pid missing"
1057 replay_barrier mds$fail_index
1058 sleep 2 # give clients a time to do operations
1059 # Increment the number of failovers
1060 num_failovers=$((num_failovers + 1))
1061 log "$TESTNAME fail mds$fail_index $num_failovers times"
1063 if (( fail_index < MDSCOUNT )); then
1064 fail_index=$((fail_index + 1))
1070 # stop the client loads
1071 kill -0 $tar_26_pid || error "tar $tar_26_pid stopped"
1072 kill -0 $dbench_26_pid || error "dbench $dbench_26_pid stopped"
1076 run_test 26 "dbench and tar with mds failover"
1079 $LFS setstripe -i 0 -c 1 $DIR2/$tfile
1080 dd if=/dev/zero of=$DIR2/$tfile bs=4096 count=1
1082 #define OBD_FAIL_LDLM_SRV_BL_AST 0x324
1083 do_facet ost1 $LCTL set_param fail_loc=0x80000324
1085 dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1 &
1089 #define OBD_FAIL_LDLM_GRANT_CHECK 0x32a
1090 do_facet ost1 $LCTL set_param fail_loc=0x32a
1095 cancel_lru_locks OST0000-osc
1096 wait $pid || error "dd failed"
1098 run_test 28 "lock replay should be ordered: waiting after granted"
1101 local dir0=$DIR/$tdir/d0
1102 local dir1=$DIR/$tdir/d1
1104 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
1105 [ $CLIENTCOUNT -lt 2 ] && skip "needs >= 2 clients" && return 0
1106 [ "$CLIENT1" == "$CLIENT2" ] &&
1107 skip "clients must be on different nodes" && return 0
1110 $LFS mkdir -i0 $dir0
1111 $LFS mkdir -i1 $dir1
1115 # create a remote dir, drop reply
1116 #define OBD_FAIL_PTLRPC_ROUND_XID 0x530
1117 $LCTL set_param fail_loc=0x530 fail_val=36
1118 #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
1119 do_facet mds2 $LCTL set_param fail_loc=0x8000015a
1120 echo make remote dir d0 for $dir0
1121 $LFS mkdir -i1 -c1 $dir0/d3 &
1124 echo make local dir d1 for $dir1
1125 do_node $CLIENT2 $LCTL set_param fail_loc=0x530 fail_val=36
1126 do_node $CLIENT2 mkdir $dir1/d4
1130 run_test 29 "replay vs update with the same xid"
1133 $LFS setstripe -E 1m -L mdt -E -1 $DIR/$tfile
1134 #first write to have no problems with grants
1135 dd if=/dev/zero of=$DIR/$tfile bs=4k count=10 ||
1136 error "dd on client failed"
1137 dd if=/dev/zero of=$DIR/$tfile bs=4k count=10 seek=10 ||
1138 error "dd on client failed"
1140 #define OBD_FAIL_LDLM_REPLAY_PAUSE 0x32e
1141 lctl set_param fail_loc=0x32e fail_val=4
1142 dd of=/dev/null if=$DIR2/$tfile &
1148 wait $pid || error "dd on client failed"
1150 run_test 30 "layout lock replay is not blocked on IO"
1153 mkdir_on_mdt0 $DIR1/$tdir
1154 $LFS setstripe -c 1 -i 0 $DIR1/$tdir
1155 for (( i=0; i < 10; i++ )) ; do
1156 mkdir -p $DIR1/$tdir/d.${i}
1158 mkdir $DIR1/$tdir/mdtdir
1159 $LFS setstripe -E 1M -L mdt $DIR1/$tdir/mdtdir
1161 # failover has to take longer than blocking timeout involved
1162 # by second multiop below which is set to obd_timeout/2 by
1164 local timeout=$(do_facet mds1 $LCTL get_param -n timeout)
1166 timeout=$((timeout / 2 + 5))
1167 fail ost1 $timeout &
1172 # consume preallocated objects, precreate thread will be awakened
1173 consume_precreations $DIR1/$tdir mds1 0 1
1175 # disable AT so that blocking timeout gets set to obd_timeout/2
1176 local amm=$(at_max_get mds1)
1179 stack_trap "at_max_set $amm mds1"
1183 #define OBD_FAIL_LLITE_XATTR_PAUSE 0x1420
1184 $LCTL set_param fail_loc=0x80001420
1185 $MULTIOP $DIR1/$tdir/mdtdir/$tfile Osw4096c &
1187 while [ ! -f $DIR1/$tdir/mdtdir/$tfile ]; do
1190 $MULTIOP $DIR2/$tdir/mdtdir/$tfile oO_WRONLY:w4096c &
1193 local mmrif=$($LCTL get_param -n \
1194 mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight | tail -1)
1195 # these are blocked by precreation until ost failover is in progress
1196 for (( i=0; i < $mmrif; i++ )) ; do
1197 $MULTIOP $DIR1/$tdir/d.${i}/parallel Oc &
1203 echo "pids: ${multiops[@]}"
1204 for pid in "${multiops[@]}"; do
1208 if (( $rc != 0 )); then
1209 echo "wait $pid failed, rc = $rc"
1213 ((failed == 0)) || error "$failed multiops failed"
1215 run_test 31 "deadlock on file_remove_privs and occupied mod rpc slots"
1218 (( $MDSCOUNT < 2 )) && skip_env "needs >= 2 MDTs"
1220 # inject a gap with 10th transaction
1221 #define OBD_FAIL_LLOG_ADD_GAP 0x131d
1222 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0000131d fail_val=10
1223 for ((i=0; i < 20; i++)); do
1224 $LFS setdirstripe -i1 $DIR/$tdir-$i ||
1225 error "can't mkdir $DIR/$tdir-$i"
1228 # prevent update llog cancellation, so next boot MDS has
1229 # process the update llog with gap injected
1230 #define OBD_FAIL_TGT_TXN_NO_CANCEL 0x726
1231 $LCTL set_param fail_loc=0x726
1236 $LCTL set_param fail_loc=0
1243 local testid=$(echo $TESTNAME | tr '_' ' ')
1244 dmesg | tac | sed "/$testid/,$ d" | grep "This client was evicted" &&
1245 error "client got evicted due to aborted recovery"
1248 run_test 32 "gap in update llog shouldn't break recovery"
1250 last_rcvd_check_incompat_flag() {
1252 local flag2check="$2"
1253 local dev=$(facet_device $facet)
1256 incompat=$(do_facet $facet $LR_READER $dev |
1257 awk '/feature_incompat:/ {print $2}')
1258 echo "last_rcvd in $dev: incompat = $incompat"
1260 return $(( (incompat & flag2check) != flag2check ))
1264 test_33() { # LU-15935
1265 (( $MDS1_VERSION >= $(version_code 2.15.52.86) )) ||
1266 (( $MDS1_VERSION >= $(version_code 2.15.2) &&
1267 $MDS1_VERSION < $(version_code 2.15.50) )) ||
1268 skip "Need MDS version at least 2.15.52.86 or 2.15.2"
1270 [[ "$mds1_FSTYPE" == "ldiskfs" ]] || skip "ldiskfs only test"
1275 # check for OBD_INCOMPAT_MULTI_RPCS (0x400) in last_rcvd
1276 last_rcvd_check_incompat_flag mds1 0x400 ||
1277 error "1st failover: OBD_INCOMPAT_MULTI_RPCS is not set on MDT0000 last_rcvd"
1279 # lose 1 client while the MDT failover
1283 wait_clients_import_state "$HOSTNAME" mds1 "\(REPLAY_WAIT\|REPLAY_LOCKS\)"
1285 do_facet mds1 $LCTL --device $(convert_facet2label mds1) abort_recovery
1286 wait_clients_import_state "$HOSTNAME" mds1 "FULL"
1289 last_rcvd_check_incompat_flag mds1 0x400 ||
1290 error "2sd failover: OBD_INCOMPAT_MULTI_RPCS is not set on MDT0000 last_rcvd"
1293 zconf_mount $HOSTNAME $MOUNT2
1294 wait_clients_import_state "$HOSTNAME" mds1 "FULL"
1296 run_test 33 "Check for OBD_INCOMPAT_MULTI_RPCS in last_rcvd after abort_recovery"
1298 complete_test $SECONDS
1299 SLEEP=$((SECONDS - $NOW))
1300 [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
1301 [ "$MOUNTED2" = yes ] && zconf_umount $HOSTNAME $MOUNT2 || true
1302 check_and_cleanup_lustre