5 LUSTRE=${LUSTRE:-$(dirname $0)/..}
6 . $LUSTRE/tests/test-framework.sh
10 ALWAYS_EXCEPT="$REPLAY_SINGLE_EXCEPT "
11 # bug number for skipped test: LU-13614
14 if [ "$mds1_FSTYPE" = zfs ]; then
19 # bug number for skipped tests: LU-9795
20 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 121"
22 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
26 CHECK_GRANT=${CHECK_GRANT:-"yes"}
27 GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
29 require_dsh_mds || exit 0
30 check_and_setup_lustre
35 rm -rf $DIR/[df][0-9]* $DIR/f.$TESTSUITE.*
37 # LU-482 Avert LVM and VM inability to flush caches in pre .33 kernels
38 if [ $LINUX_VERSION_CODE -lt $(version_code 2.6.33) ]; then
40 do_facet $SINGLEMDS sync
43 test_0a() { # was test_0
44 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
45 replay_barrier $SINGLEMDS
49 run_test 0a "empty replay"
52 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
54 # this test attempts to trigger a race in the precreation code,
55 # and must run before any other objects are created on the filesystem
57 createmany -o $DIR/$tfile 20 || error "createmany -o $DIR/$tfile failed"
58 unlinkmany $DIR/$tfile 20 || error "unlinkmany $DIR/$tfile failed"
60 run_test 0b "ensure object created after recover exists. (3284)"
63 replay_barrier $SINGLEMDS
66 facet_failover $SINGLEMDS
67 zconf_mount $(hostname) $MOUNT || error "mount fails"
68 client_up || error "post-failover df failed"
69 # file shouldn't exist if replay-barrier works as expected
70 rm $DIR/$tfile && error "File exists and it shouldn't"
73 run_test 0c "check replay-barrier"
76 replay_barrier $SINGLEMDS
78 facet_failover $SINGLEMDS
79 zconf_mount $(hostname) $MOUNT || error "mount fails"
80 client_up || error "post-failover df failed"
82 run_test 0d "expired recovery with no clients"
85 replay_barrier $SINGLEMDS
88 $CHECKSTAT -t file $DIR/$tfile ||
89 error "$CHECKSTAT $DIR/$tfile attribute check failed"
92 run_test 1 "simple create"
95 replay_barrier $SINGLEMDS
98 $CHECKSTAT -t file $DIR/$tfile ||
99 error "$CHECKSTAT $DIR/$tfile attribute check failed"
105 mcreate $DIR/$tfile || error "mcreate $DIR/$tfile failed"
106 replay_barrier $SINGLEMDS
109 $CHECKSTAT -t file $DIR/$tfile ||
110 error "$CHECKSTAT $DIR/$tfile attribute check failed"
116 replay_barrier $SINGLEMDS
117 $LFS setstripe -c $OSTCOUNT $DIR/$tfile
119 $CHECKSTAT -t file $DIR/$tfile ||
120 error "$CHECKSTAT $DIR/$tfile check failed"
122 run_test 2c "setstripe replay"
125 [[ "$mds1_FSTYPE" = zfs ]] &&
126 [[ "$MDS1_VERSION" -lt $(version_code 2.12.51) ]] &&
127 skip "requires LU-10143 fix on MDS"
128 replay_barrier $SINGLEMDS
129 $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir
131 $CHECKSTAT -t dir $DIR/$tdir ||
132 error "$CHECKSTAT $DIR/$tdir check failed"
134 run_test 2d "setdirstripe replay"
137 testid=$(echo $TESTNAME | tr '_' ' ')
138 #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
139 do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000013b"
140 openfile -f O_CREAT:O_EXCL $DIR/$tfile &
142 replay_barrier $SINGLEMDS
145 $CHECKSTAT -t file $DIR/$tfile ||
146 error "$CHECKSTAT $DIR/$tfile attribute check failed"
147 dmesg | tac | sed "/$testid/,$ d" | \
148 grep "Open request replay failed with -17" &&
149 error "open replay failed" || true
151 run_test 2e "O_CREAT|O_EXCL create replay"
154 local file=$DIR/$tfile
155 replay_barrier $SINGLEMDS
157 openfile -f O_DIRECTORY $file
159 $CHECKSTAT -t file $file ||
160 error "$CHECKSTAT $file attribute check failed"
163 run_test 3a "replay failed open(O_DIRECTORY)"
166 replay_barrier $SINGLEMDS
167 #define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE
168 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000114"
170 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
172 $CHECKSTAT -t file $DIR/$tfile &&
173 error "$CHECKSTAT $DIR/$tfile attribute check should fail"
176 run_test 3b "replay failed open -ENOMEM"
179 replay_barrier $SINGLEMDS
180 #define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE
181 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000128"
183 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
186 $CHECKSTAT -t file $DIR/$tfile &&
187 error "$CHECKSTAT $DIR/$tfile attribute check should fail"
190 run_test 3c "replay failed open -ENOMEM"
192 test_4a() { # was test_4
193 replay_barrier $SINGLEMDS
194 for i in $(seq 10); do
195 echo "tag-$i" > $DIR/$tfile-$i
198 for i in $(seq 10); do
199 grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
202 run_test 4a "|x| 10 open(O_CREAT)s"
205 for i in $(seq 10); do
206 echo "tag-$i" > $DIR/$tfile-$i
208 replay_barrier $SINGLEMDS
211 $CHECKSTAT -t file $DIR/$tfile-* &&
212 error "$CHECKSTAT $DIR/$tfile-* attribute check should fail" ||
215 run_test 4b "|x| rm 10 files"
217 # The idea is to get past the first block of precreated files on both
218 # osts, and then replay.
220 replay_barrier $SINGLEMDS
221 for i in $(seq 220); do
222 echo "tag-$i" > $DIR/$tfile-$i
225 for i in $(seq 220); do
226 grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
230 # waiting for commitment of removal
232 run_test 5 "|x| 220 open(O_CREAT)"
234 test_6a() { # was test_6
235 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
236 replay_barrier $SINGLEMDS
237 mcreate $DIR/$tdir/$tfile
239 $CHECKSTAT -t dir $DIR/$tdir ||
240 error "$CHECKSTAT $DIR/$tdir attribute check failed"
241 $CHECKSTAT -t file $DIR/$tdir/$tfile ||
242 error "$CHECKSTAT $DIR/$tdir/$tfile attribute check failed"
244 # waiting for log process thread
246 run_test 6a "mkdir + contained create"
249 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
250 replay_barrier $SINGLEMDS
253 $CHECKSTAT -t dir $DIR/$tdir &&
254 error "$CHECKSTAT $DIR/$tdir attribute check should fail" ||
257 run_test 6b "|X| rmdir"
260 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
261 replay_barrier $SINGLEMDS
262 mcreate $DIR/$tdir/$tfile
264 $CHECKSTAT -t dir $DIR/$tdir ||
265 error "$CHECKSTAT $DIR/$tdir attribute check failed"
266 $CHECKSTAT -t file $DIR/$tdir/$tfile ||
267 error "$CHECKSTAT $DIR/$tdir/$tfile attribute check failed"
270 run_test 7 "mkdir |X| contained create"
273 replay_barrier $SINGLEMDS
274 multiop_bg_pause $DIR/$tfile mo_c ||
275 error "multiop mknod $DIR/$tfile failed"
279 $CHECKSTAT -t file $DIR/$tfile ||
280 error "$CHECKSTAT $DIR/$tfile attribute check failed"
281 kill -USR1 $MULTIPID || error "multiop mknod $MULTIPID not running"
282 wait $MULTIPID || error "multiop mknod $MULTIPID failed"
285 run_test 8 "creat open |X| close"
288 replay_barrier $SINGLEMDS
290 local old_inum=$(ls -i $DIR/$tfile | awk '{print $1}')
292 local new_inum=$(ls -i $DIR/$tfile | awk '{print $1}')
294 echo " old_inum == $old_inum, new_inum == $new_inum"
295 if [ $old_inum -eq $new_inum ] ;
297 echo "old_inum and new_inum match"
299 echo " old_inum and new_inum do not match"
300 error "old index($old_inum) does not match new index($new_inum)"
304 run_test 9 "|X| create (same inum/gen)"
307 mcreate $DIR/$tfile || error "mcreate $DIR/$tfile failed"
308 replay_barrier $SINGLEMDS
309 mv $DIR/$tfile $DIR/$tfile-2
312 $CHECKSTAT $DIR/$tfile &&
313 error "$CHECKSTAT $DIR/$tfile attribute check should fail"
314 $CHECKSTAT $DIR/$tfile-2 ||
315 error "$CHECKSTAT $DIR/$tfile-2 attribute check failed"
319 run_test 10 "create |X| rename unlink"
322 mcreate $DIR/$tfile || error "mcreate $DIR/$tfile failed"
323 echo "old" > $DIR/$tfile
324 mv $DIR/$tfile $DIR/$tfile-2
325 replay_barrier $SINGLEMDS
326 echo "new" > $DIR/$tfile
328 grep old $DIR/$tfile-2
330 grep new $DIR/$tfile || error "grep $DIR/$tfile failed"
331 grep old $DIR/$tfile-2 || error "grep $DIR/$tfile-2 failed"
333 run_test 11 "create open write rename |X| create-old-name read"
336 mcreate $DIR/$tfile || error "mcreate $DIR/$tfile failed"
337 multiop_bg_pause $DIR/$tfile o_tSc ||
338 error "multiop_bg_pause $DIR/$tfile failed"
341 replay_barrier $SINGLEMDS
342 kill -USR1 $pid || error "multiop $pid not running"
343 wait $pid || error "multiop $pid failed"
346 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
349 run_test 12 "open, unlink |X| close"
351 # 1777 - replay open after committed chmod that would make
352 # a regular open a failure
354 mcreate $DIR/$tfile || error "mcreate $DIR/$tfile failed"
355 multiop_bg_pause $DIR/$tfile O_wc ||
356 error "multiop_bg_pause $DIR/$tfile failed"
359 $CHECKSTAT -p 0 $DIR/$tfile ||
360 error "$CHECKSTAT $DIR/$tfile attribute check failed"
361 replay_barrier $SINGLEMDS
363 kill -USR1 $pid || error "multiop $pid not running"
364 wait $pid || error "multiop $pid failed"
366 $CHECKSTAT -s 1 -p 0 $DIR/$tfile ||
367 error "second $CHECKSTAT $DIR/$tfile attribute check failed"
368 rm $DIR/$tfile || error "rm $DIR/$tfile failed"
371 run_test 13 "open chmod 0 |x| write close"
374 multiop_bg_pause $DIR/$tfile O_tSc ||
375 error "multiop_bg_pause $DIR/$tfile failed"
378 replay_barrier $SINGLEMDS
379 kill -USR1 $pid || error "multiop $pid not running"
380 wait $pid || error "multiop $pid failed"
383 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
386 run_test 14 "open(O_CREAT), unlink |X| close"
389 multiop_bg_pause $DIR/$tfile O_tSc ||
390 error "multiop_bg_pause $DIR/$tfile failed"
393 replay_barrier $SINGLEMDS
394 touch $DIR/$tfile-1 || error "touch $DIR/$tfile-1 failed"
395 kill -USR1 $pid || error "multiop $pid not running"
396 wait $pid || error "multiop $pid failed"
399 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
400 touch $DIR/$tfile-2 || error "touch $DIR/$tfile-2 failed"
403 run_test 15 "open(O_CREAT), unlink |X| touch new, close"
406 replay_barrier $SINGLEMDS
409 mcreate $DIR/$tfile-2
411 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
412 [ -e $DIR/$tfile-2 ] || error "file $DIR/$tfile-2 does not exist"
413 munlink $DIR/$tfile-2 || error "munlink $DIR/$tfile-2 failed"
415 run_test 16 "|X| open(O_CREAT), unlink, touch new, unlink new"
418 replay_barrier $SINGLEMDS
419 multiop_bg_pause $DIR/$tfile O_c ||
420 error "multiop_bg_pause $DIR/$tfile failed"
423 kill -USR1 $pid || error "multiop $pid not running"
424 wait $pid || error "multiop $pid failed"
425 $CHECKSTAT -t file $DIR/$tfile ||
426 error "$CHECKSTAT $DIR/$tfile attribute check failed"
429 run_test 17 "|X| open(O_CREAT), |replay| close"
432 replay_barrier $SINGLEMDS
433 multiop_bg_pause $DIR/$tfile O_tSc ||
434 error "multiop_bg_pause $DIR/$tfile failed"
437 touch $DIR/$tfile-2 || error "touch $DIR/$tfile-2 failed"
438 echo "pid: $pid will close"
439 kill -USR1 $pid || error "multiop $pid not running"
440 wait $pid || error "multiop $pid failed"
443 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
444 [ -e $DIR/$tfile-2 ] || error "file $DIR/$tfile-2 does not exist"
445 # this touch frequently fails
446 touch $DIR/$tfile-3 || error "touch $DIR/$tfile-3 failed"
447 munlink $DIR/$tfile-2 || error "munlink $DIR/$tfile-2 failed"
448 munlink $DIR/$tfile-3 || error "munlink $DIR/$tfile-3 failed"
451 run_test 18 "open(O_CREAT), unlink, touch new, close, touch, unlink"
453 # bug 1855 (a simpler form of test_11 above)
455 replay_barrier $SINGLEMDS
457 echo "old" > $DIR/$tfile
458 mv $DIR/$tfile $DIR/$tfile-2
459 grep old $DIR/$tfile-2
461 grep old $DIR/$tfile-2 || error "grep $DIR/$tfile-2 failed"
463 run_test 19 "mcreate, open, write, rename "
465 test_20a() { # was test_20
466 replay_barrier $SINGLEMDS
467 multiop_bg_pause $DIR/$tfile O_tSc ||
468 error "multiop_bg_pause $DIR/$tfile failed"
473 kill -USR1 $pid || error "multiop $pid not running"
474 wait $pid || error "multiop $pid failed"
475 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
478 run_test 20a "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
480 test_20b() { # bug 10480
481 local wait_timeout=$((TIMEOUT * 4))
482 local extra=$(fs_log_size)
486 save_layout_restore_at_exit $MOUNT
487 $LFS setstripe -i 0 -c 1 $DIR
489 local beforeused=$(df -P $DIR | tail -1 | awk '{ print $3 }')
491 dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
492 while [ ! -e $DIR/$tfile ] ; do
493 sleep 0.01 # give dd a chance to start
496 $LFS getstripe $DIR/$tfile || error "$LFS getstripe $DIR/$tfile failed"
498 rm -f $DIR/$tfile || error "rm -f $DIR/$tfile failed"
500 client_up || client_up || true # reconnect
502 do_facet $SINGLEMDS "lctl set_param -n osd*.*MDT*.force_sync=1"
504 fail $SINGLEMDS # start orphan recovery
505 wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
506 wait_delete_completed $wait_timeout || error "delete did not finish"
510 local afterused=$(df -P $DIR | tail -1 | awk '{ print $3 }')
511 log "before $beforeused, after $afterused"
513 (( $beforeused + $extra >= $afterused )) && break
514 n_attempts=$((n_attempts + 1))
515 [ $n_attempts -gt 3 ] &&
516 error "after $afterused > before $beforeused + $extra"
518 wait_zfs_commit $SINGLEMDS 5
523 run_test 20b "write, unlink, eviction, replay (test mds_cleanup_orphans)"
525 test_20c() { # bug 10480
526 multiop_bg_pause $DIR/$tfile Ow_c ||
527 error "multiop_bg_pause $DIR/$tfile failed"
533 client_up || client_up || true # reconnect
535 kill -USR1 $pid || error "multiop $pid not running"
536 wait $pid || error "multiop $pid failed"
537 [ -s $DIR/$tfile ] || error "File was truncated"
541 run_test 20c "check that client eviction does not affect file content"
544 replay_barrier $SINGLEMDS
545 multiop_bg_pause $DIR/$tfile O_tSc ||
546 error "multiop_bg_pause $DIR/$tfile failed"
549 touch $DIR/$tfile-1 || error "touch $DIR/$tfile-1 failed"
552 kill -USR1 $pid || error "multiop $pid not running"
553 wait $pid || error "multiop $pid failed"
554 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
555 touch $DIR/$tfile-2 || error "touch $DIR/$tfile-2 failed"
558 run_test 21 "|X| open(O_CREAT), unlink touch new, replay, close (test mds_cleanup_orphans)"
561 multiop_bg_pause $DIR/$tfile O_tSc ||
562 error "multiop_bg_pause $DIR/$tfile failed"
565 replay_barrier $SINGLEMDS
569 kill -USR1 $pid || error "multiop $pid not running"
570 wait $pid || error "multiop $pid failed"
571 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
574 run_test 22 "open(O_CREAT), |X| unlink, replay, close (test mds_cleanup_orphans)"
577 multiop_bg_pause $DIR/$tfile O_tSc ||
578 error "multiop_bg_pause $DIR/$tfile failed"
581 replay_barrier $SINGLEMDS
583 touch $DIR/$tfile-1 || error "touch $DIR/$tfile-1 failed"
586 kill -USR1 $pid || error "multiop $pid not running"
587 wait $pid || error "multiop $pid failed"
588 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
589 touch $DIR/$tfile-2 || error "touch $DIR/$tfile-2 failed"
592 run_test 23 "open(O_CREAT), |X| unlink touch new, replay, close (test mds_cleanup_orphans)"
595 multiop_bg_pause $DIR/$tfile O_tSc ||
596 error "multiop_bg_pause $DIR/$tfile failed"
599 replay_barrier $SINGLEMDS
602 kill -USR1 $pid || error "multiop $pid not running"
603 wait $pid || error "multiop $pid failed"
604 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
607 run_test 24 "open(O_CREAT), replay, unlink, close (test mds_cleanup_orphans)"
610 multiop_bg_pause $DIR/$tfile O_tSc ||
611 error "multiop_bg_pause $DIR/$tfile failed"
615 replay_barrier $SINGLEMDS
617 kill -USR1 $pid || error "multiop $pid not running"
618 wait $pid || error "multiop $pid failed"
619 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
622 run_test 25 "open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
625 replay_barrier $SINGLEMDS
626 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
627 error "multiop_bg_pause $DIR/$tfile-1 failed"
629 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
630 error "multiop_bg_pause $DIR/$tfile-2 failed"
634 kill -USR1 $pid2 || error "second multiop $pid2 not running"
635 wait $pid2 || error "second multiop $pid2 failed"
638 kill -USR1 $pid1 || error "multiop $pid1 not running"
639 wait $pid1 || error "multiop $pid1 failed"
640 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
641 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
644 run_test 26 "|X| open(O_CREAT), unlink two, close one, replay, close one (test mds_cleanup_orphans)"
647 replay_barrier $SINGLEMDS
648 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
649 error "multiop_bg_pause $DIR/$tfile-1 failed"
651 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
652 error "multiop_bg_pause $DIR/$tfile-2 failed"
658 kill -USR1 $pid1 || error "multiop $pid1 not running"
659 wait $pid1 || error "multiop $pid1 failed"
660 kill -USR1 $pid2 || error "second multiop $pid2 not running"
661 wait $pid2 || error "second multiop $pid2 failed"
662 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
663 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
666 run_test 27 "|X| open(O_CREAT), unlink two, replay, close two (test mds_cleanup_orphans)"
669 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
670 error "multiop_bg_pause $DIR/$tfile-1 failed"
672 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
673 error "multiop_bg_pause $DIR/$tfile-2 failed"
675 replay_barrier $SINGLEMDS
678 kill -USR1 $pid2 || error "second multiop $pid2 not running"
679 wait $pid2 || error "second multiop $pid2 failed"
682 kill -USR1 $pid1 || error "multiop $pid1 not running"
683 wait $pid1 || error "multiop $pid1 failed"
684 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
685 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
688 run_test 28 "open(O_CREAT), |X| unlink two, close one, replay, close one (test mds_cleanup_orphans)"
691 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
692 error "multiop_bg_pause $DIR/$tfile-1 failed"
694 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
695 error "multiop_bg_pause $DIR/$tfile-2 failed"
697 replay_barrier $SINGLEMDS
702 kill -USR1 $pid1 || error "multiop $pid1 not running"
703 wait $pid1 || error "multiop $pid1 failed"
704 kill -USR1 $pid2 || error "second multiop $pid2 not running"
705 wait $pid2 || error "second multiop $pid2 failed"
706 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
707 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
710 run_test 29 "open(O_CREAT), |X| unlink two, replay, close two (test mds_cleanup_orphans)"
713 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
714 error "multiop_bg_pause $DIR/$tfile-1 failed"
716 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
717 error "multiop_bg_pause $DIR/$tfile-2 failed"
722 replay_barrier $SINGLEMDS
724 kill -USR1 $pid1 || error "multiop $pid1 not running"
725 wait $pid1 || error "multiop $pid1 failed"
726 kill -USR1 $pid2 || error "second multiop $pid2 not running"
727 wait $pid2 || error "second multiop $pid2 failed"
728 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
729 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
732 run_test 30 "open(O_CREAT) two, unlink two, replay, close two (test mds_cleanup_orphans)"
735 multiop_bg_pause $DIR/$tfile-1 O_tSc ||
736 error "multiop_bg_pause $DIR/$tfile-1 failed"
738 multiop_bg_pause $DIR/$tfile-2 O_tSc ||
739 error "multiop_bg_pause $DIR/$tfile-2 failed"
743 replay_barrier $SINGLEMDS
746 kill -USR1 $pid1 || error "multiop $pid1 not running"
747 wait $pid1 || error "multiop $pid1 failed"
748 kill -USR1 $pid2 || error "second multiop $pid2 not running"
749 wait $pid2 || error "second multiop $pid2 failed"
750 [ -e $DIR/$tfile-1 ] && error "file $DIR/$tfile-1 should not exist"
751 [ -e $DIR/$tfile-2 ] && error "file $DIR/$tfile-2 should not exist"
754 run_test 31 "open(O_CREAT) two, unlink one, |X| unlink one, close two (test mds_cleanup_orphans)"
756 # tests for bug 2104; completion without crashing is success. The close is
757 # stale, but we always return 0 for close, so the app never sees it.
759 multiop_bg_pause $DIR/$tfile O_c ||
760 error "multiop_bg_pause $DIR/$tfile failed"
762 multiop_bg_pause $DIR/$tfile O_c ||
763 error "second multiop_bg_pause $DIR/$tfile failed"
766 client_up || client_up || error "client_up failed"
767 kill -USR1 $pid1 || error "multiop $pid1 not running"
768 kill -USR1 $pid2 || error "second multiop $pid2 not running"
769 wait $pid1 || error "multiop $pid1 failed"
770 wait $pid2 || error "second multiop $pid2 failed"
773 run_test 32 "close() notices client eviction; close() after client eviction"
776 createmany -o $DIR/$tfile-%d 10 ||
777 error "createmany create $DIR/$tfile failed"
778 replay_barrier_nosync $SINGLEMDS
779 fail_abort $SINGLEMDS
780 # recreate shouldn't fail
781 createmany -o $DIR/$tfile--%d 10 ||
782 error "createmany recreate $DIR/$tfile failed"
786 run_test 33a "fid seq shouldn't be reused after abort recovery"
789 #define OBD_FAIL_SEQ_ALLOC 0x1311
790 do_facet $SINGLEMDS "lctl set_param fail_loc=0x1311"
792 createmany -o $DIR/$tfile-%d 10
793 replay_barrier_nosync $SINGLEMDS
794 fail_abort $SINGLEMDS
795 # recreate shouldn't fail
796 createmany -o $DIR/$tfile--%d 10 ||
797 error "createmany recreate $DIR/$tfile failed"
801 run_test 33b "test fid seq allocation"
804 multiop_bg_pause $DIR/$tfile O_c ||
805 error "multiop_bg_pause $DIR/$tfile failed"
809 replay_barrier $SINGLEMDS
810 fail_abort $SINGLEMDS
811 kill -USR1 $pid || error "multiop $pid not running"
812 wait $pid || error "multiop $pid failed"
813 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
817 run_test 34 "abort recovery before client does replay (test mds_cleanup_orphans)"
819 # bug 2278 - generate one orphan on OST, then destroy it during recovery from llog
821 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
823 #define OBD_FAIL_MDS_REINT_NET_REP 0x119
824 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000119"
829 # give a chance to remove from MDS
830 fail_abort $SINGLEMDS
831 $CHECKSTAT -t file $DIR/$tfile &&
832 error "$CHECKSTAT $DIR/$tfile attribute check should fail" ||
835 run_test 35 "test recovery from llog for unlink op"
837 # b=2432 resent cancel after replay uses wrong cookie,
838 # so don't resend cancels
840 replay_barrier $SINGLEMDS
842 checkstat $DIR/$tfile
843 facet_failover $SINGLEMDS
845 if $LCTL dk | grep "stale lock .*cookie"; then
846 error "cancel after replay failed"
849 run_test 36 "don't resend cancel"
852 # directory orphans can't be unlinked from PENDING directory
854 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $tdir failed"
855 rmdir $DIR/$tdir/$tfile 2>/dev/null
856 multiop_bg_pause $DIR/$tdir/$tfile dD_c ||
857 error "multiop_bg_pause $tfile failed"
859 rmdir $DIR/$tdir/$tfile
861 replay_barrier $SINGLEMDS
862 # clear the dmesg buffer so we only see errors from this recovery
863 do_facet $SINGLEMDS dmesg -c >/dev/null
864 fail_abort $SINGLEMDS
865 kill -USR1 $pid || error "multiop $pid not running"
866 do_facet $SINGLEMDS dmesg | grep "error unlinking orphan" &&
867 error "error unlinking files"
868 wait $pid || error "multiop $pid failed"
872 run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans for directories)"
875 createmany -o $DIR/$tfile-%d 800 ||
876 error "createmany -o $DIR/$tfile failed"
877 unlinkmany $DIR/$tfile-%d 0 400 || error "unlinkmany $DIR/$tfile failed"
878 replay_barrier $SINGLEMDS
880 unlinkmany $DIR/$tfile-%d 400 400 ||
881 error "unlinkmany $DIR/$tfile 400 failed"
883 $CHECKSTAT -t file $DIR/$tfile-* &&
884 error "$CHECKSTAT $DIR/$tfile-* attribute check should fail" ||
887 run_test 38 "test recovery from unlink llog (test llog_gen_rec) "
889 test_39() { # bug 4176
890 createmany -o $DIR/$tfile-%d 800 ||
891 error "createmany -o $DIR/$tfile failed"
892 replay_barrier $SINGLEMDS
893 unlinkmany $DIR/$tfile-%d 0 400
895 unlinkmany $DIR/$tfile-%d 400 400 ||
896 error "unlinkmany $DIR/$tfile 400 failed"
898 $CHECKSTAT -t file $DIR/$tfile-* &&
899 error "$CHECKSTAT $DIR/$tfile-* attribute check should fail" ||
902 run_test 39 "test recovery from unlink llog (test llog_gen_rec) "
905 lctl get_param -n osc.*.stats | awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }'
910 # always need connection to MDS to verify layout during IO. LU-2628.
911 lctl get_param mdc.*.connect_flags | grep -q layout_lock &&
912 skip "layout_lock needs MDS connection for IO" && return 0
914 $LCTL mark multiop $MOUNT/$tfile OS_c
915 multiop $MOUNT/$tfile OS_c &
917 writeme -s $MOUNT/${tfile}-2 &
920 facet_failover $SINGLEMDS
921 #define OBD_FAIL_MDS_CONNECT_NET 0x117
922 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000117"
924 stat1=$(count_ost_writes)
926 stat2=$(count_ost_writes)
927 echo "$stat1, $stat2"
928 if [ $stat1 -lt $stat2 ]; then
929 echo "writes continuing during recovery"
932 echo "writes not continuing during recovery, bug 2477"
935 echo "waiting for writeme $WRITE_PID"
939 echo "waiting for multiop $PID"
940 wait $PID || error "multiop $PID failed"
941 do_facet client munlink $MOUNT/$tfile ||
942 error "munlink $MOUNT/$tfile failed"
943 do_facet client munlink $MOUNT/${tfile}-2 ||
944 error "munlink $MOUNT/$tfile-2 failed"
947 run_test 40 "cause recovery in ptlrpc, ensure IO continues"
950 # make sure that a read to one osc doesn't try to double-unlock its page just
951 # because another osc is invalid. trigger_group_io used to mistakenly return
952 # an error if any oscs were invalid even after having successfully put rpcs
953 # on valid oscs. This was fatal if the caller was ll_readpage who unlocked
954 # the page, guarnateeing that the unlock from the RPC completion would
955 # assert on trying to unlock the unlocked page.
957 [ $OSTCOUNT -lt 2 ] && skip_env "needs >= 2 OSTs" && return
959 local f=$MOUNT/$tfile
960 # make sure the start of the file is ost1
961 $LFS setstripe -S $((128 * 1024)) -i 0 $f
962 do_facet client dd if=/dev/zero of=$f bs=4k count=1 ||
963 error "dd on client failed"
965 # fail ost2 and read from ost1
966 local mdtosc=$(get_mdtosc_proc_path $SINGLEMDS $ost2_svc)
967 local osc2dev=$(do_facet $SINGLEMDS "lctl get_param -n devices" |
968 grep $mdtosc | awk '{print $1}')
969 [ -z "$osc2dev" ] && echo "OST: $ost2_svc" &&
970 lctl get_param -n devices &&
971 error "OST 2 $osc2dev does not exist"
972 do_facet $SINGLEMDS $LCTL --device $osc2dev deactivate ||
973 error "deactive device on $SINGLEMDS failed"
974 do_facet client dd if=$f of=/dev/null bs=4k count=1 ||
975 error "second dd on client failed"
976 do_facet $SINGLEMDS $LCTL --device $osc2dev activate ||
977 error "active device on $SINGLEMDS failed"
980 run_test 41 "read from a valid osc while other oscs are invalid"
982 # test MDS recovery after ost failure
984 blocks=$(df -P $MOUNT | tail -n 1 | awk '{ print $2 }')
985 createmany -o $DIR/$tfile-%d 800 ||
986 error "createmany -o $DIR/$tfile failed"
988 unlinkmany $DIR/$tfile-%d 0 400
990 lctl set_param debug=-1
993 # osc is evicted, fs is smaller (but only with failout OSTs (bug 7287)
994 #blocks_after=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'`
995 #[ $blocks_after -lt $blocks ] || return 1
996 echo "wait for MDS to timeout and recover"
997 sleep $((TIMEOUT * 2))
999 unlinkmany $DIR/$tfile-%d 400 400 ||
1000 error "unlinkmany $DIR/$tfile 400 failed"
1001 $CHECKSTAT -t file $DIR/$tfile-* &&
1002 error "$CHECKSTAT $DIR/$tfile-* attribute check should fail" ||
1005 run_test 42 "recovery after ost failure"
1007 # timeout in MDS/OST recovery RPC will LBUG MDS
1008 test_43() { # bug 2530
1009 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1011 replay_barrier $SINGLEMDS
1013 # OBD_FAIL_OST_CREATE_NET 0x204
1014 do_facet ost1 "lctl set_param fail_loc=0x80000204"
1020 run_test 43 "mds osc import failure during recovery; don't LBUG"
1022 test_44a() { # was test_44
1023 local at_max_saved=0
1025 local mdcdev=$($LCTL dl |
1026 awk "/${FSNAME}-MDT0000-mdc-/ {if (\$2 == \"UP\") {print \$1}}")
1027 [ "$mdcdev" ] || error "${FSNAME}-MDT0000-mdc- not UP"
1028 [ $(echo $mdcdev | wc -w) -eq 1 ] ||
1029 { $LCTL dl; error "looking for mdcdev=$mdcdev"; }
1031 # adaptive timeouts slow this way down
1032 if at_is_enabled; then
1033 at_max_saved=$(at_max_get mds)
1037 for i in $(seq 1 10); do
1038 echo "$i of 10 ($(date +%s))"
1039 do_facet $SINGLEMDS \
1040 "lctl get_param -n md[ts].*.mdt.timeouts | grep service"
1041 #define OBD_FAIL_TGT_CONN_RACE 0x701
1042 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000701"
1043 # lctl below may fail, it is valid case
1044 $LCTL --device $mdcdev recover
1047 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1048 [ $at_max_saved -ne 0 ] && at_max_set $at_max_saved mds
1051 run_test 44a "race in target handle connect"
1054 local mdcdev=$($LCTL dl |
1055 awk "/${FSNAME}-MDT0000-mdc-/ {if (\$2 == \"UP\") {print \$1}}")
1056 [ "$mdcdev" ] || error "${FSNAME}-MDT0000-mdc not up"
1057 [ $(echo $mdcdev | wc -w) -eq 1 ] ||
1058 { echo mdcdev=$mdcdev; $LCTL dl;
1059 error "more than one ${FSNAME}-MDT0000-mdc"; }
1061 for i in $(seq 1 10); do
1062 echo "$i of 10 ($(date +%s))"
1063 do_facet $SINGLEMDS \
1064 "lctl get_param -n md[ts].*.mdt.timeouts | grep service"
1065 #define OBD_FAIL_TGT_DELAY_RECONNECT 0x704
1066 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000704"
1067 # lctl below may fail, it is valid case
1068 $LCTL --device $mdcdev recover
1073 run_test 44b "race in target handle connect"
1076 replay_barrier $SINGLEMDS
1077 createmany -m $DIR/$tfile-%d 100 || error "failed to create directories"
1078 #define OBD_FAIL_TGT_RCVG_FLAG 0x712
1079 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000712"
1080 fail_abort $SINGLEMDS
1081 unlinkmany $DIR/$tfile-%d 100 && error "unliked after fail abort"
1083 unlinkmany $DIR/$tfile-%d 100 && error "unliked after fail"
1086 run_test 44c "race in target handle connect"
1088 # Handle failed close
1090 local mdcdev=$($LCTL get_param -n devices |
1091 awk "/ ${FSNAME}-MDT0000-mdc-/ {print \$1}")
1092 [ "$mdcdev" ] || error "${FSNAME}-MDT0000-mdc not up"
1093 [ $(echo $mdcdev | wc -w) -eq 1 ] ||
1094 { echo mdcdev=$mdcdev; $LCTL dl;
1095 error "more than one ${FSNAME}-MDT0000-mdc"; }
1097 $LCTL --device $mdcdev recover ||
1098 error "$LCTL --device $mdcdev recover failed"
1100 multiop_bg_pause $DIR/$tfile O_c ||
1101 error "multiop_bg_pause $DIR/$tfile failed"
1104 # This will cause the CLOSE to fail before even
1105 # allocating a reply buffer
1106 $LCTL --device $mdcdev deactivate ||
1107 error "$LCTL --device $mdcdev deactivate failed"
1110 kill -USR1 $pid || error "multiop $pid not running"
1111 wait $pid || error "multiop $pid failed"
1113 $LCTL --device $mdcdev activate ||
1114 error "$LCTL --device $mdcdev activate failed"
1117 $CHECKSTAT -t file $DIR/$tfile ||
1118 error "$CHECKSTAT $DIR/$tfile attribute check failed"
1121 run_test 45 "Handle failed close"
1124 drop_reply "touch $DIR/$tfile"
1126 # ironically, the previous test, 45, will cause a real forced close,
1127 # so just look for one for this test
1128 local FID=$($LFS path2fid $tfile)
1129 $LCTL dk | grep -i "force closing file handle $FID" &&
1130 error "found force closing in dmesg"
1133 run_test 46 "Don't leak file handle after open resend (3325)"
1135 test_47() { # bug 2824
1136 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1138 # create some files to make sure precreate has been done on all
1139 # OSTs. (just in case this test is run independently)
1140 createmany -o $DIR/$tfile 20 ||
1141 error "createmany create $DIR/$tfile failed"
1143 # OBD_FAIL_OST_CREATE_NET 0x204
1145 do_facet ost1 "lctl set_param fail_loc=0x80000204"
1146 client_up || error "client_up failed"
1148 # let the MDS discover the OST failure, attempt to recover, fail
1149 # and recover again.
1150 sleep $((3 * TIMEOUT))
1152 # Without 2824, this createmany would hang
1153 createmany -o $DIR/$tfile 20 ||
1154 error "createmany recraete $DIR/$tfile failed"
1155 unlinkmany $DIR/$tfile 20 || error "unlinkmany $DIR/$tfile failed"
1159 run_test 47 "MDS->OSC failure during precreate cleanup (2824)"
1162 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1163 [ "$OSTCOUNT" -lt "2" ] && skip_env "needs >= 2 OSTs" && return
1165 replay_barrier $SINGLEMDS
1166 createmany -o $DIR/$tfile 20 ||
1167 error "createmany -o $DIR/$tfile failed"
1168 # OBD_FAIL_OST_EROFS 0x216
1169 facet_failover $SINGLEMDS
1170 do_facet ost1 "lctl set_param fail_loc=0x80000216"
1171 client_up || error "client_up failed"
1173 # let the MDS discover the OST failure, attempt to recover, fail
1174 # and recover again.
1175 sleep $((3 * TIMEOUT))
1177 createmany -o $DIR/$tfile 20 20 ||
1178 error "createmany recraete $DIR/$tfile failed"
1179 unlinkmany $DIR/$tfile 40 || error "unlinkmany $DIR/$tfile failed"
1182 run_test 48 "MDS->OSC failure during precreate cleanup (2824)"
1185 local mdtosc=$(get_mdtosc_proc_path $SINGLEMDS $ost1_svc)
1186 local oscdev=$(do_facet $SINGLEMDS "lctl get_param -n devices" |
1187 grep $mdtosc | awk '{print $1}')
1188 [ "$oscdev" ] || error "could not find OSC device on MDS"
1189 do_facet $SINGLEMDS $LCTL --device $oscdev recover ||
1190 error "OSC device $oscdev recovery failed"
1191 do_facet $SINGLEMDS $LCTL --device $oscdev recover ||
1192 error "second OSC device $oscdev recovery failed"
1193 # give the mds_lov_sync threads a chance to run
1196 run_test 50 "Double OSC recovery, don't LASSERT (3812)"
1198 # b3764 timed out lock replay
1200 [ "$MDS1_VERSION" -lt $(version_code 2.6.90) ] &&
1201 skip "MDS prior to 2.6.90 handle LDLM_REPLY_NET incorrectly"
1203 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
1204 cancel_lru_locks mdc
1206 multiop_bg_pause $DIR/$tfile s_s || error "multiop $DIR/$tfile failed"
1209 #define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
1210 lctl set_param -n ldlm.cancel_unused_locks_before_replay "0"
1211 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000157"
1213 fail $SINGLEMDS || error "fail $SINGLEMDS failed"
1215 wait $mpid || error "multiop_bg_pause pid failed"
1217 do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
1218 lctl set_param fail_loc=0x0
1219 lctl set_param -n ldlm.cancel_unused_locks_before_replay "1"
1222 run_test 52 "time out lock replay (3764)"
1224 # bug 3462 - simultaneous MDC requests
1226 [[ $(lctl get_param mdc.*.import |
1227 grep "connect_flags:.*multi_mod_rpc") ]] ||
1228 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
1230 cancel_lru_locks mdc # cleanup locks from former test cases
1231 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1232 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1233 multiop $DIR/${tdir}-1/f O_c &
1235 # give multiop a change to open
1238 #define OBD_FAIL_MDS_CLOSE_NET 0x115
1239 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
1240 kill -USR1 $close_pid
1241 cancel_lru_locks mdc # force the close
1242 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1244 mcreate $DIR/${tdir}-2/f || error "mcreate $DIR/${tdir}-2/f failed"
1246 # close should still be here
1247 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1249 replay_barrier_nodf $SINGLEMDS
1251 wait $close_pid || error "close_pid $close_pid failed"
1253 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1254 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1255 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1256 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1257 rm -rf $DIR/${tdir}-*
1259 run_test 53a "|X| close request while two MDC requests in flight"
1262 cancel_lru_locks mdc # cleanup locks from former test cases
1264 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1265 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1266 multiop_bg_pause $DIR/${tdir}-1/f O_c ||
1267 error "multiop_bg_pause $DIR/${tdir}-1/f failed"
1270 #define OBD_FAIL_MDS_REINT_NET 0x107
1271 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000107"
1272 mcreate $DIR/${tdir}-2/f &
1276 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1277 kill -USR1 $close_pid
1278 cancel_lru_locks mdc # force the close
1279 wait $close_pid || error "close_pid $close_pid failed"
1280 # open should still be here
1281 [ -d /proc/$open_pid ] || error "open_pid doesn't exist"
1283 replay_barrier_nodf $SINGLEMDS
1285 wait $open_pid || error "open_pid failed"
1287 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1288 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1289 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1290 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1291 rm -rf $DIR/${tdir}-*
1293 run_test 53b "|X| open request while two MDC requests in flight"
1296 cancel_lru_locks mdc # cleanup locks from former test cases
1298 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1299 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1300 multiop $DIR/${tdir}-1/f O_c &
1303 #define OBD_FAIL_MDS_REINT_NET 0x107
1304 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000107"
1305 mcreate $DIR/${tdir}-2/f &
1309 #define OBD_FAIL_MDS_CLOSE_NET 0x115
1310 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
1311 kill -USR1 $close_pid
1312 cancel_lru_locks mdc # force the close
1314 #bz20647: make sure all pids exist before failover
1315 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1316 [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
1317 replay_barrier_nodf $SINGLEMDS
1318 fail_nodf $SINGLEMDS
1319 wait $open_pid || error "open_pid failed"
1321 # close should be gone
1322 [ -d /proc/$close_pid ] && error "close_pid should not exist"
1323 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1325 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1326 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1327 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1328 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1329 rm -rf $DIR/${tdir}-*
1331 run_test 53c "|X| open request and close request while two MDC requests in flight"
1334 [[ $(lctl get_param mdc.*.import |
1335 grep "connect_flags:.*multi_mod_rpc") ]] ||
1336 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
1338 cancel_lru_locks mdc # cleanup locks from former test cases
1340 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1341 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1342 multiop $DIR/${tdir}-1/f O_c &
1344 # give multiop a chance to open
1347 #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
1348 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
1349 kill -USR1 $close_pid
1350 cancel_lru_locks mdc # force the close
1351 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1352 mcreate $DIR/${tdir}-2/f || error "mcreate $DIR/${tdir}-2/f failed"
1354 # close should still be here
1355 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1357 wait $close_pid || error "close_pid failed"
1359 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1360 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1361 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1362 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1363 rm -rf $DIR/${tdir}-*
1365 run_test 53d "close reply while two MDC requests in flight"
1368 cancel_lru_locks mdc # cleanup locks from former test cases
1370 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1371 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1372 multiop $DIR/${tdir}-1/f O_c &
1375 #define OBD_FAIL_MDS_REINT_NET_REP 0x119
1376 do_facet $SINGLEMDS "lctl set_param fail_loc=0x119"
1377 mcreate $DIR/${tdir}-2/f &
1381 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1382 kill -USR1 $close_pid
1383 cancel_lru_locks mdc # force the close
1384 wait $close_pid || error "close_pid failed"
1385 # open should still be here
1386 [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
1388 replay_barrier_nodf $SINGLEMDS
1390 wait $open_pid || error "open_pid failed"
1392 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1393 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1394 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1395 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1396 rm -rf $DIR/${tdir}-*
1398 run_test 53e "|X| open reply while two MDC requests in flight"
1401 cancel_lru_locks mdc # cleanup locks from former test cases
1403 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1404 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1405 multiop $DIR/${tdir}-1/f O_c &
1408 #define OBD_FAIL_MDS_REINT_NET_REP 0x119
1409 do_facet $SINGLEMDS "lctl set_param fail_loc=0x119"
1410 mcreate $DIR/${tdir}-2/f &
1414 #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
1415 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
1416 kill -USR1 $close_pid
1417 cancel_lru_locks mdc # force the close
1419 #bz20647: make sure all pids are exists before failover
1420 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1421 [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
1422 replay_barrier_nodf $SINGLEMDS
1423 fail_nodf $SINGLEMDS
1424 wait $open_pid || error "open_pid failed"
1426 # close should be gone
1427 [ -d /proc/$close_pid ] && error "close_pid should not exist"
1428 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1430 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1431 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1432 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1433 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1434 rm -rf $DIR/${tdir}-*
1436 run_test 53f "|X| open reply and close reply while two MDC requests in flight"
1439 cancel_lru_locks mdc # cleanup locks from former test cases
1441 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1442 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1443 multiop $DIR/${tdir}-1/f O_c &
1446 #define OBD_FAIL_MDS_REINT_NET_REP 0x119
1447 do_facet $SINGLEMDS "lctl set_param fail_loc=0x119"
1448 mcreate $DIR/${tdir}-2/f &
1452 #define OBD_FAIL_MDS_CLOSE_NET 0x115
1453 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
1454 kill -USR1 $close_pid
1455 cancel_lru_locks mdc # force the close
1456 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1458 #bz20647: make sure all pids are exists before failover
1459 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1460 [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
1461 replay_barrier_nodf $SINGLEMDS
1462 fail_nodf $SINGLEMDS
1463 wait $open_pid || error "open_pid failed"
1465 # close should be gone
1466 [ -d /proc/$close_pid ] && error "close_pid should not exist"
1468 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1469 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1470 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1471 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1472 rm -rf $DIR/${tdir}-*
1474 run_test 53g "|X| drop open reply and close request while close and open are both in flight"
1477 cancel_lru_locks mdc # cleanup locks from former test cases
1479 mkdir_on_mdt0 $DIR/${tdir}-1 || error "mkdir $DIR/${tdir}-1 failed"
1480 mkdir_on_mdt0 $DIR/${tdir}-2 || error "mkdir $DIR/${tdir}-2 failed"
1481 multiop $DIR/${tdir}-1/f O_c &
1484 #define OBD_FAIL_MDS_REINT_NET 0x107
1485 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000107"
1486 mcreate $DIR/${tdir}-2/f &
1490 #define OBD_FAIL_MDS_CLOSE_NET_REP 0x13b
1491 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013b"
1492 kill -USR1 $close_pid
1493 cancel_lru_locks mdc # force the close
1496 #bz20647: make sure all pids are exists before failover
1497 [ -d /proc/$close_pid ] || error "close_pid doesn't exist"
1498 [ -d /proc/$open_pid ] || error "open_pid doesn't exists"
1499 replay_barrier_nodf $SINGLEMDS
1500 fail_nodf $SINGLEMDS
1501 wait $open_pid || error "open_pid failed"
1503 # close should be gone
1504 [ -d /proc/$close_pid ] && error "close_pid should not exist"
1505 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1507 $CHECKSTAT -t file $DIR/${tdir}-1/f ||
1508 error "$CHECKSTAT $DIR/${tdir}-1/f attribute check failed"
1509 $CHECKSTAT -t file $DIR/${tdir}-2/f ||
1510 error "$CHECKSTAT $DIR/${tdir}-2/f attribute check failed"
1511 rm -rf $DIR/${tdir}-*
1513 run_test 53h "open request and close reply while two MDC requests in flight"
1515 #b3761 ASSERTION(hash != 0) failed
1517 # OBD_FAIL_MDS_OPEN_CREATE | OBD_FAIL_ONCE
1518 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000012b"
1520 # give touch a chance to run
1522 do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
1526 run_test 55 "let MDS_CHECK_RESENT return the original return code instead of 0"
1528 #b3440 ASSERTION(rec->ur_fid2->id) failed
1530 ln -s foo $DIR/$tfile
1531 replay_barrier $SINGLEMDS
1532 #drop_reply "cat $DIR/$tfile"
1536 run_test 56 "don't replay a symlink open request (3440)"
1538 #recovery one mds-ost setattr from llog
1540 #define OBD_FAIL_MDS_OST_SETATTR 0x12c
1541 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000012c"
1542 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
1543 replay_barrier $SINGLEMDS
1545 wait_recovery_complete $SINGLEMDS || error "MDS recovery is not done"
1546 wait_mds_ost_sync || error "wait_mds_ost_sync failed"
1547 $CHECKSTAT -t file $DIR/$tfile ||
1548 error "$CHECKSTAT $DIR/$tfile attribute check failed"
1549 do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
1552 run_test 57 "test recovery from llog for setattr op"
1555 zconf_umount $(hostname) $MOUNT2
1559 #recovery many mds-ost setattr from llog
1561 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1562 #define OBD_FAIL_MDS_OST_SETATTR 0x12c
1563 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000012c"
1564 createmany -o $DIR/$tdir/$tfile-%d 2500
1565 replay_barrier $SINGLEMDS
1568 $CHECKSTAT -t file $DIR/$tdir/$tfile-* >/dev/null ||
1569 error "$CHECKSTAT $DIR/$tfile-* attribute check failed"
1570 do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
1571 unlinkmany $DIR/$tdir/$tfile-%d 2500 ||
1572 error "unlinkmany $DIR/$tfile failed"
1575 run_test 58a "test recovery from llog for setattr op (test llog_gen_rec)"
1581 trap cleanup_58 EXIT
1583 large_xattr_enabled &&
1584 orig="$(generate_string $(max_xattr_size))" || orig="bar"
1585 # Original extended attribute can be long. Print a small version of
1586 # attribute if an error occurs
1587 local sm_msg=$(printf "%.9s" $orig)
1589 mount_client $MOUNT2 || error "mount_client on $MOUNT2 failed"
1590 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1591 touch $DIR/$tdir/$tfile || error "touch $DIR/$tdir/$tfile failed"
1592 replay_barrier $SINGLEMDS
1593 setfattr -n trusted.foo -v $orig $DIR/$tdir/$tfile
1595 new=$(get_xattr_value trusted.foo $MOUNT2/$tdir/$tfile)
1596 [[ "$new" = "$orig" ]] ||
1597 error "xattr set ($sm_msg...) differs from xattr get ($new)"
1598 rm -f $DIR/$tdir/$tfile
1601 wait_clients_import_state ${CLIENTS:-$HOSTNAME} "mgs" FULL
1603 run_test 58b "test replay of setxattr op"
1605 test_58c() { # bug 16570
1610 trap cleanup_58 EXIT
1612 if large_xattr_enabled; then
1613 local xattr_size=$(max_xattr_size)
1614 orig="$(generate_string $((xattr_size / 2)))"
1615 orig1="$(generate_string $xattr_size)"
1621 # PING_INTERVAL max(obd_timeout / 4, 1U)
1622 sleep $((TIMEOUT / 4))
1624 # Original extended attribute can be long. Print a small version of
1625 # attribute if an error occurs
1626 local sm_msg=$(printf "%.9s" $orig)
1627 local sm_msg1=$(printf "%.9s" $orig1)
1629 mount_client $MOUNT2 || error "mount_client on $MOUNT2 failed"
1630 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1631 touch $DIR/$tdir/$tfile || error "touch $DIR/$tdir/$tfile failed"
1632 drop_request "setfattr -n trusted.foo -v $orig $DIR/$tdir/$tfile" ||
1633 error "drop_request for setfattr failed"
1634 new=$(get_xattr_value trusted.foo $MOUNT2/$tdir/$tfile)
1635 [[ "$new" = "$orig" ]] ||
1636 error "xattr set ($sm_msg...) differs from xattr get ($new)"
1637 drop_reint_reply "setfattr -n trusted.foo1 \
1638 -v $orig1 $DIR/$tdir/$tfile" ||
1639 error "drop_reint_reply for setfattr failed"
1640 new=$(get_xattr_value trusted.foo1 $MOUNT2/$tdir/$tfile)
1641 [[ "$new" = "$orig1" ]] ||
1642 error "second xattr set ($sm_msg1...) differs xattr get ($new)"
1643 rm -f $DIR/$tdir/$tfile
1647 run_test 58c "resend/reconstruct setxattr op"
1649 # log_commit_thread vs filter_destroy race used to lead to import use after free
1652 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1654 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1655 createmany -o $DIR/$tdir/$tfile-%d 200 ||
1656 error "createmany create files failed"
1658 unlinkmany $DIR/$tdir/$tfile-%d 200 ||
1659 error "unlinkmany $DIR/$tdir/$tfile failed"
1660 #define OBD_FAIL_PTLRPC_DELAY_RECOV 0x507
1661 do_facet ost1 "lctl set_param fail_loc=0x507"
1664 do_facet ost1 "lctl set_param fail_loc=0x0"
1668 run_test 59 "test log_commit_thread vs filter_destroy race"
1670 # race between add unlink llog vs cat log init in post_recovery (only for b1_6)
1671 # bug 12086: should no oops and No ctxt error for this test
1673 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1674 createmany -o $DIR/$tdir/$tfile-%d 200 ||
1675 error "createmany create files failed"
1676 replay_barrier $SINGLEMDS
1677 unlinkmany $DIR/$tdir/$tfile-%d 0 100
1679 unlinkmany $DIR/$tdir/$tfile-%d 100 100
1680 local no_ctxt=$(dmesg | grep "No ctxt")
1681 [ -z "$no_ctxt" ] || error "ctxt is not initialized in recovery"
1683 run_test 60 "test llog post recovery init vs llog unlink"
1685 #test race llog recovery thread vs llog cleanup
1686 test_61a() { # was test_61
1687 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1689 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1690 createmany -o $DIR/$tdir/$tfile-%d 800 ||
1691 error "createmany create files failed"
1693 unlinkmany $DIR/$tdir/$tfile-%d 800
1694 # OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221
1695 set_nodes_failloc "$(osts_nodes)" 0x80000221
1700 set_nodes_failloc "$(osts_nodes)" 0x0
1702 $CHECKSTAT -t file $DIR/$tdir/$tfile-* &&
1703 error "$CHECKSTAT $DIR/$tdir/$tfile attribute check should fail"
1706 run_test 61a "test race llog recovery vs llog cleanup"
1708 #test race mds llog sync vs llog cleanup
1710 # OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x13a
1711 do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000013a"
1712 facet_failover $SINGLEMDS
1715 do_facet client dd if=/dev/zero of=$DIR/$tfile bs=4k count=1 ||
1718 run_test 61b "test race mds llog sync vs llog cleanup"
1720 #test race cancel cookie cb vs llog cleanup
1722 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1724 # OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222
1725 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
1726 set_nodes_failloc "$(osts_nodes)" 0x80000222
1730 set_nodes_failloc "$(osts_nodes)" 0x0
1732 run_test 61c "test race mds llog sync vs llog cleanup"
1734 test_61d() { # bug 16002 # bug 17466 # bug 22137
1735 # OBD_FAIL_OBD_LLOG_SETUP 0x605
1737 do_facet mgs "lctl set_param fail_loc=0x80000605"
1738 start mgs $(mgsdevname) $MGS_MOUNT_OPTS &&
1739 error "mgs start should have failed"
1740 do_facet mgs "lctl set_param fail_loc=0"
1741 start mgs $(mgsdevname) $MGS_MOUNT_OPTS || error "cannot restart mgs"
1743 run_test 61d "error in llog_setup should cleanup the llog context correctly"
1745 test_62() { # Bug 15756 - don't mis-drop resent replay
1746 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1747 replay_barrier $SINGLEMDS
1748 createmany -o $DIR/$tdir/$tfile- 25 ||
1749 error "createmany create files failed"
1750 #define OBD_FAIL_TGT_REPLAY_DROP 0x707
1751 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000707"
1753 do_facet $SINGLEMDS "lctl set_param fail_loc=0"
1754 unlinkmany $DIR/$tdir/$tfile- 25 ||
1755 error "unlinkmany $DIR/$tdir/$tfile failed"
1758 run_test 62 "don't mis-drop resent replay"
1760 #Adaptive Timeouts (bug 3055)
1768 echo "Cleaning up AT ..."
1769 if [ -n "$ATOLDBASE" ]; then
1770 local at_history=$($LCTL get_param -n at_history)
1771 do_facet $SINGLEMDS "lctl set_param at_history=$at_history" || true
1772 do_facet ost1 "lctl set_param at_history=$at_history" || true
1775 if [ $AT_MAX_SET -ne 0 ]; then
1776 for facet in mds client ost; do
1777 var=AT_MAX_SAVE_${facet}
1778 echo restore AT on $facet to saved value ${!var}
1779 at_max_set ${!var} $facet
1780 at_new=$(at_max_get $facet)
1781 echo Restored AT value on $facet $at_new
1782 [ $at_new -eq ${!var} ] ||
1783 error "AT value not restored SAVED ${!var} NEW $at_new"
1790 local at_max_new=600
1792 # Save at_max original values
1794 if [ $AT_MAX_SET -eq 0 ]; then
1795 # Suppose that all osts have the same at_max
1796 for facet in mds client ost; do
1797 eval AT_MAX_SAVE_${facet}=$(at_max_get $facet)
1801 for facet in mds client ost; do
1802 at_max=$(at_max_get $facet)
1803 if [ $at_max -ne $at_max_new ]; then
1804 echo "AT value on $facet is $at_max, set it by force temporarily to $at_max_new"
1805 at_max_set $at_max_new $facet
1810 if [ -z "$ATOLDBASE" ]; then
1811 ATOLDBASE=$(do_facet $SINGLEMDS "lctl get_param -n at_history")
1812 # speed up the timebase so we can check decreasing AT
1813 do_facet $SINGLEMDS "lctl set_param at_history=8" || true
1814 do_facet ost1 "lctl set_param at_history=8" || true
1816 # sleep for a while to cool down, should be > 8s and also allow
1817 # at least one ping to be sent. simply use TIMEOUT to be safe.
1822 test_65a() #bug 3055
1824 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1826 at_start || return 0
1827 $LCTL dk > /dev/null
1829 $LCTL set_param debug="other"
1830 # Slow down a request to the current service time, this is critical
1831 # because previous tests may have caused this value to increase.
1832 REQ_DELAY=`lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts |
1833 awk '/portal 12/ {print $5}'`
1834 REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5))
1836 do_facet $SINGLEMDS lctl set_param fail_val=$((${REQ_DELAY} * 1000))
1837 #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
1838 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x8000050a
1839 createmany -o $DIR/$tfile 10 > /dev/null
1840 unlinkmany $DIR/$tfile 10 > /dev/null
1841 # check for log message
1842 $LCTL dk | grep -i "Early reply #" || error "No early reply"
1844 # client should show REQ_DELAY estimates
1845 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep portal
1847 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep portal
1849 run_test 65a "AT: verify early replies"
1851 test_65b() #bug 3055
1853 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1855 at_start || return 0
1858 $LCTL set_param debug="other trace"
1859 $LCTL dk > /dev/null
1860 # Slow down a request to the current service time, this is critical
1861 # because previous tests may have caused this value to increase.
1862 $LFS setstripe --stripe-index=0 --stripe-count=1 $DIR/$tfile ||
1863 error "$LFS setstripe failed for $DIR/$tfile"
1865 multiop $DIR/$tfile Ow1yc
1866 REQ_DELAY=`lctl get_param -n osc.${FSNAME}-OST0000-osc-*.timeouts |
1867 awk '/portal 6/ {print $5}'`
1868 REQ_DELAY=$((${REQ_DELAY} + ${REQ_DELAY} / 4 + 5))
1870 do_facet ost1 lctl set_param fail_val=${REQ_DELAY}
1871 #define OBD_FAIL_OST_BRW_PAUSE_PACK 0x224
1872 do_facet ost1 $LCTL set_param fail_loc=0x224
1875 $LFS setstripe --stripe-index=0 --stripe-count=1 $DIR/$tfile ||
1876 error "$LFS setstripe failed"
1877 # force some real bulk transfer
1878 multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
1880 do_facet ost1 $LCTL set_param fail_loc=0
1881 # check for log message
1882 $LCTL dk | grep -i "Early reply #" || error "No early reply"
1884 # client should show REQ_DELAY estimates
1885 lctl get_param -n osc.${FSNAME}-OST0000-osc-*.timeouts | grep portal
1887 run_test 65b "AT: verify early replies on packed reply / bulk"
1889 test_66a() #bug 3055
1891 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1893 at_start || return 0
1894 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
1895 # adjust 5s at a time so no early reply is sent (within deadline)
1896 do_facet $SINGLEMDS "$LCTL set_param fail_val=5000"
1897 #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
1898 do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
1899 createmany -o $DIR/$tfile 20 > /dev/null
1900 unlinkmany $DIR/$tfile 20 > /dev/null
1901 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
1902 do_facet $SINGLEMDS "$LCTL set_param fail_val=10000"
1903 do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
1904 createmany -o $DIR/$tfile 20 > /dev/null
1905 unlinkmany $DIR/$tfile 20 > /dev/null
1906 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
1907 do_facet $SINGLEMDS "$LCTL set_param fail_loc=0"
1909 createmany -o $DIR/$tfile 20 > /dev/null
1910 unlinkmany $DIR/$tfile 20 > /dev/null
1911 lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
1912 CUR=$(lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | awk '/portal 12/ {print $5}')
1913 WORST=$(lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | awk '/portal 12/ {print $7}')
1914 echo "Current MDT timeout $CUR, worst $WORST"
1915 [ $CUR -lt $WORST ] || error "Current $CUR should be less than worst $WORST"
1917 run_test 66a "AT: verify MDT service time adjusts with no early replies"
1919 test_66b() #bug 3055
1921 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1923 at_start || return 0
1924 ORIG=$(lctl get_param -n mdc.${FSNAME}-MDT0000*.timeouts |
1925 awk '/network/ {print $4}')
1926 $LCTL set_param fail_val=$(($ORIG + 5))
1927 #define OBD_FAIL_PTLRPC_PAUSE_REP 0x50c
1928 $LCTL set_param fail_loc=0x50c
1929 touch $DIR/$tfile > /dev/null 2>&1
1930 $LCTL set_param fail_loc=0
1931 CUR=$(lctl get_param -n mdc.${FSNAME}-MDT0000*.timeouts |
1932 awk '/network/ {print $4}')
1933 WORST=$(lctl get_param -n mdc.${FSNAME}-MDT0000*.timeouts |
1934 awk '/network/ {print $6}')
1935 echo "network timeout orig $ORIG, cur $CUR, worst $WORST"
1936 [ $WORST -gt $ORIG ] ||
1937 error "Worst $WORST should be worse than orig $ORIG"
1939 run_test 66b "AT: verify net latency adjusts"
1941 test_67a() #bug 3055
1943 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1945 at_start || return 0
1946 CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
1947 # sleeping threads may drive values above this
1948 do_facet ost1 "$LCTL set_param fail_val=400"
1949 #define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
1950 do_facet ost1 "$LCTL set_param fail_loc=0x50a"
1951 createmany -o $DIR/$tfile 20 > /dev/null
1952 unlinkmany $DIR/$tfile 20 > /dev/null
1953 do_facet ost1 "$LCTL set_param fail_loc=0"
1954 CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
1955 ATTEMPTS=$(($CONN2 - $CONN1))
1956 echo "$ATTEMPTS osc reconnect attempts on gradual slow"
1957 [ $ATTEMPTS -gt 0 ] &&
1958 error_ignore bz13721 "AT should have prevented reconnect"
1961 run_test 67a "AT: verify slow request processing doesn't induce reconnects"
1963 test_67b() #bug 3055
1965 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
1967 at_start || return 0
1968 CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
1970 # exhaust precreations on ost1
1971 local OST=$(ostname_from_index 0)
1972 local mdtosc=$(get_mdtosc_proc_path mds $OST)
1973 local last_id=$(do_facet $SINGLEMDS lctl get_param -n \
1974 osp.$mdtosc.prealloc_last_id)
1975 local next_id=$(do_facet $SINGLEMDS lctl get_param -n \
1976 osp.$mdtosc.prealloc_next_id)
1978 mkdir -p $DIR/$tdir/${OST} || error "mkdir $DIR/$tdir/${OST} failed"
1979 $LFS setstripe -i 0 -c 1 $DIR/$tdir/${OST} ||
1980 error "$LFS setstripe failed"
1981 echo "Creating to objid $last_id on ost $OST..."
1982 #define OBD_FAIL_OST_PAUSE_CREATE 0x223
1983 do_facet ost1 "$LCTL set_param fail_val=20000"
1984 do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
1985 createmany -o $DIR/$tdir/${OST}/f $next_id $((last_id - next_id + 2))
1988 do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
1990 CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
1991 ATTEMPTS=$(($CONN2 - $CONN1))
1992 echo "$ATTEMPTS osc reconnect attempts on instant slow"
1993 # do it again; should not timeout
1994 do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
1995 cp /etc/profile $DIR/$tfile || error "cp failed"
1996 do_facet ost1 "$LCTL set_param fail_loc=0"
1998 do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
1999 CONN3=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
2000 ATTEMPTS=$(($CONN3 - $CONN2))
2001 echo "$ATTEMPTS osc reconnect attempts on 2nd slow"
2002 [ $ATTEMPTS -gt 0 ] && error "AT should have prevented reconnect"
2005 run_test 67b "AT: verify instant slowdown doesn't induce reconnects"
2007 test_68 () #bug 13813
2009 remote_ost_nodsh && skip "remote OST with nodsh" && return 0
2011 at_start || return 0
2012 local ldlm_enqueue_min=$(find /sys -name ldlm_enqueue_min)
2013 [ -z "$ldlm_enqueue_min" ] && skip "missing /sys/.../ldlm_enqueue_min" && return 0
2014 local ldlm_enqueue_min_r=$(do_facet ost1 "find /sys -name ldlm_enqueue_min")
2015 [ -z "$ldlm_enqueue_min_r" ] && skip "missing /sys/.../ldlm_enqueue_min in the ost1" && return 0
2016 local ENQ_MIN=$(cat $ldlm_enqueue_min)
2017 local ENQ_MIN_R=$(do_facet ost1 "cat $ldlm_enqueue_min_r")
2018 echo $TIMEOUT >> $ldlm_enqueue_min
2019 do_facet ost1 "echo $TIMEOUT >> $ldlm_enqueue_min_r"
2021 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2022 $LFS setstripe --stripe-index=0 -c 1 $DIR/$tdir ||
2023 error "$LFS setstripe failed for $DIR/$tdir"
2024 #define OBD_FAIL_LDLM_PAUSE_CANCEL 0x312
2025 $LCTL set_param fail_val=$(($TIMEOUT - 1))
2026 $LCTL set_param fail_loc=0x80000312
2027 cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?"
2028 $LCTL set_param fail_val=$((TIMEOUT * 5 / 4))
2029 $LCTL set_param fail_loc=0x80000312
2030 cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?"
2031 $LCTL set_param fail_loc=0
2033 echo $ENQ_MIN >> $ldlm_enqueue_min
2034 do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r"
2038 run_test 68 "AT: verify slowing locks"
2041 # end of AT tests includes above lines
2043 # start multi-client tests
2045 [ -z "$CLIENTS" ] &&
2046 { skip "Need two or more clients." && return; }
2047 [ $CLIENTCOUNT -lt 2 ] &&
2048 { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
2050 echo "mount clients $CLIENTS ..."
2051 zconf_mount_clients $CLIENTS $MOUNT
2053 local clients=${CLIENTS//,/ }
2054 echo "Write/read files on $DIR ; clients $CLIENTS ... "
2055 for CLIENT in $clients; do
2056 do_node $CLIENT dd bs=1M count=10 if=/dev/zero \
2057 of=$DIR/${tfile}_${CLIENT} 2>/dev/null ||
2058 error "dd failed on $CLIENT"
2061 local prev_client=$(echo $clients | sed 's/^.* \(.\+\)$/\1/')
2062 for C in ${CLIENTS//,/ }; do
2063 do_node $prev_client dd if=$DIR/${tfile}_${C} \
2064 of=/dev/null 2>/dev/null ||
2065 error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
2071 run_test 70a "check multi client t-f"
2073 check_for_process () {
2078 killall_process $clients "$prog" -0
2082 local clients=${CLIENTS:-$HOSTNAME}
2084 zconf_mount_clients $clients $MOUNT
2087 [ "$SLOW" = "no" ] && duration=120
2088 # set duration to 900 because it takes some time to boot node
2089 [ "$FAILURE_MODE" = HARD ] && duration=900
2092 local start_ts=$(date +%s)
2093 local cmd="rundbench 1 -t $duration"
2095 if [ $MDSCOUNT -ge 2 ]; then
2096 test_mkdir -p -c$MDSCOUNT $DIR/$tdir
2097 $LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir
2099 do_nodesv $clients "set -x; MISSING_DBENCH_OK=$MISSING_DBENCH_OK \
2100 PATH=\$PATH:$LUSTRE/utils:$LUSTRE/tests/:$DBENCH_LIB \
2101 DBENCH_LIB=$DBENCH_LIB TESTSUITE=$TESTSUITE TESTNAME=$TESTNAME \
2102 MOUNT=$MOUNT DIR=$DIR/$tdir/\\\$(hostname) LCTL=$LCTL $cmd" &
2105 #LU-1897 wait for all dbench copies to start
2106 while ! check_for_process $clients dbench; do
2107 elapsed=$(($(date +%s) - start_ts))
2108 if [ $elapsed -gt $duration ]; then
2109 killall_process $clients dbench
2110 error "dbench failed to start on $clients!"
2115 log "Started rundbench load pid=$pid ..."
2117 elapsed=$(($(date +%s) - start_ts))
2118 local num_failovers=0
2120 while [ $elapsed -lt $duration ]; do
2121 if ! check_for_process $clients dbench; then
2122 error_noexit "dbench stopped on some of $clients!"
2123 killall_process $clients dbench
2127 replay_barrier mds$fail_index
2128 sleep 1 # give clients a time to do operations
2129 # Increment the number of failovers
2130 num_failovers=$((num_failovers+1))
2131 log "$TESTNAME fail mds$fail_index $num_failovers times"
2133 elapsed=$(($(date +%s) - start_ts))
2134 if [ $fail_index -ge $MDSCOUNT ]; then
2137 fail_index=$((fail_index+1))
2141 wait $pid || error "rundbench load on $clients failed!"
2143 run_test 70b "dbench ${MDSCOUNT}mdts recovery; $CLIENTCOUNT clients"
2144 # end multi-client tests
2149 local monitor_pid=$3
2151 local start_ts=$(date +%s)
2152 local num_failovers=0
2155 elapsed=$(($(date +%s) - start_ts))
2156 while [ $elapsed -lt $duration ]; do
2157 fail_index=$((RANDOM%max_index+1))
2158 kill -0 $monitor_pid ||
2159 error "$monitor_pid stopped"
2161 replay_barrier mds$fail_index
2163 # Increment the number of failovers
2164 num_failovers=$((num_failovers+1))
2165 log "$TESTNAME fail mds$fail_index $num_failovers times"
2167 elapsed=$(($(date +%s) - start_ts))
2173 rm -f $DIR/replay-single.70c.lck
2178 local clients=${CLIENTS:-$HOSTNAME}
2181 zconf_mount_clients $clients $MOUNT
2184 [ "$SLOW" = "no" ] && duration=180
2185 # set duration to 900 because it takes some time to boot node
2186 [ "$FAILURE_MODE" = HARD ] && duration=600
2189 local start_ts=$(date +%s)
2191 trap cleanup_70c EXIT
2193 while [ ! -e $DIR/replay-single.70c.lck ]; do
2194 test_mkdir -p -c$MDSCOUNT $DIR/$tdir || break
2195 if [ $MDSCOUNT -ge 2 ]; then
2196 $LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir ||
2197 error "set default dirstripe failed"
2199 cd $DIR/$tdir || break
2200 tar cf - /etc | tar xf - || error "tar failed in loop"
2204 echo "Started tar $tar_70c_pid"
2206 random_fail_mdt $MDSCOUNT $duration $tar_70c_pid
2207 kill -0 $tar_70c_pid || error "tar $tar_70c_pid stopped"
2209 touch $DIR/replay-single.70c.lck
2210 wait $tar_70c_pid || error "$?: tar failed"
2215 run_test 70c "tar ${MDSCOUNT}mdts recovery"
2219 kill -9 $mkdir_70d_pid
2223 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2224 local clients=${CLIENTS:-$HOSTNAME}
2227 zconf_mount_clients $clients $MOUNT
2230 [ "$SLOW" = "no" ] && duration=180
2231 # set duration to 900 because it takes some time to boot node
2232 [ "$FAILURE_MODE" = HARD ] && duration=900
2237 local start_ts=$(date +%s)
2239 trap cleanup_70d EXIT
2242 $LFS mkdir -i0 -c2 $DIR/$tdir/test || {
2246 $LFS mkdir -i1 -c2 $DIR/$tdir/test1 || {
2251 touch $DIR/$tdir/test/a || {
2255 mkdir $DIR/$tdir/test/b || {
2259 rm -rf $DIR/$tdir/test || {
2265 touch $DIR/$tdir/test1/a || {
2269 mkdir $DIR/$tdir/test1/b || {
2274 rm -rf $DIR/$tdir/test1 || {
2276 ls -lR $DIR/$tdir/test1
2282 echo "Started $mkdir_70d_pid"
2284 random_fail_mdt $MDSCOUNT $duration $mkdir_70d_pid
2285 kill -0 $mkdir_70d_pid || error "mkdir/rmdir $mkdir_70d_pid stopped"
2290 run_test 70d "mkdir/rmdir striped dir ${MDSCOUNT}mdts recovery"
2293 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2294 local clients=${CLIENTS:-$HOSTNAME}
2297 lctl set_param debug=+ha
2298 zconf_mount_clients $clients $MOUNT
2301 [ "$SLOW" = "no" ] && duration=180
2302 # set duration to 900 because it takes some time to boot node
2303 [ "$FAILURE_MODE" = HARD ] && duration=900
2306 $LFS mkdir -i0 $DIR/$tdir/test_0
2307 $LFS mkdir -i0 $DIR/$tdir/test_1
2308 touch $DIR/$tdir/test_0/a
2309 touch $DIR/$tdir/test_1/b
2312 mrename $DIR/$tdir/test_0/a $DIR/$tdir/test_1/b > /dev/null || {
2317 checkstat $DIR/$tdir/test_0/a && {
2318 echo "a still exists"
2322 checkstat $DIR/$tdir/test_1/b || {
2323 echo "b still exists"
2327 touch $DIR/$tdir/test_0/a || {
2328 echo "touch a fails"
2332 mrename $DIR/$tdir/test_1/b $DIR/$tdir/test_0/a > /dev/null || {
2339 stack_trap "kill -9 $rename_70e_pid" EXIT
2340 echo "Started PID=$rename_70e_pid"
2342 random_fail_mdt 2 $duration $rename_70e_pid
2343 kill -0 $rename_70e_pid || error "rename $rename_70e_pid stopped"
2345 run_test 70e "rename cross-MDT with random fails"
2347 test_70f_write_and_read(){
2352 echo "Write/read files in: '$DIR/$tdir', clients: '$CLIENTS' ..."
2353 for client in ${CLIENTS//,/ }; do
2354 [ -f $stopflag ] || return
2356 local tgtfile=$DIR/$tdir/$tfile.$client
2357 do_node $client dd $DD_OPTS bs=1M count=10 if=$srcfile \
2358 of=$tgtfile 2>/dev/null ||
2359 error "dd $DD_OPTS bs=1M count=10 if=$srcfile " \
2360 "of=$tgtfile failed on $client, rc=$?"
2363 local prev_client=$(echo ${CLIENTS//,/ } | awk '{ print $NF }')
2366 for client in ${CLIENTS//,/ }; do
2367 [ -f $stopflag ] || return
2369 # flush client cache in case test is running on only one client
2370 # do_node $client cancel_lru_locks osc
2371 do_node $client $LCTL set_param ldlm.namespaces.*.lru_size=clear
2373 tgtfile=$DIR/$tdir/$tfile.$client
2374 local md5=$(do_node $prev_client "md5sum $tgtfile")
2375 [ ${checksum[$index]// */} = ${md5// */} ] ||
2376 error "$tgtfile: checksum doesn't match on $prev_client"
2377 index=$((index + 1))
2387 mkdir -p $DIR/$tdir || error "cannot create $DIR/$tdir directory"
2388 $LFS setstripe -c -1 $DIR/$tdir ||
2389 error "cannot $LFS setstripe $DIR/$tdir"
2392 while [ -f $stopflag ]; do
2393 test_70f_write_and_read $srcfile $stopflag
2394 # use direct IO and buffer cache in turns if loop
2395 [ -n "$DD_OPTS" ] && DD_OPTS="" || DD_OPTS="oflag=direct"
2399 test_70f_cleanup() {
2401 rm -f $TMP/$tfile.stop
2402 do_nodes $CLIENTS rm -f $TMP/$tfile
2403 rm -f $DIR/$tdir/$tfile.*
2407 # [ x$ost1failover_HOST = x$ost_HOST ] &&
2408 # { skip "Failover host not defined" && return; }
2409 # [ -z "$CLIENTS" ] &&
2410 # { skip "CLIENTS are not specified." && return; }
2411 # [ $CLIENTCOUNT -lt 2 ] &&
2412 # { skip "Need 2 or more clients, have $CLIENTCOUNT" && return; }
2414 [[ "$OST1_VERSION" -lt $(version_code 2.9.53) ]] &&
2415 skip "Need server version at least 2.9.53"
2417 echo "mount clients $CLIENTS ..."
2418 zconf_mount_clients $CLIENTS $MOUNT
2420 local srcfile=$TMP/$tfile
2424 trap test_70f_cleanup EXIT
2425 # create a different source file local to each client node so we can
2426 # detect if the file wasn't written out properly after failover
2427 do_nodes $CLIENTS dd bs=1M count=10 if=/dev/urandom of=$srcfile \
2428 2>/dev/null || error "can't create $srcfile on $CLIENTS"
2429 for client in ${CLIENTS//,/ }; do
2430 checksum[$index]=$(do_node $client "md5sum $srcfile")
2431 index=$((index + 1))
2435 [ "$SLOW" = "no" ] && duration=60
2436 # set duration to 900 because it takes some time to boot node
2437 [ "$FAILURE_MODE" = HARD ] && duration=900
2439 local stopflag=$TMP/$tfile.stop
2440 test_70f_loop $srcfile $stopflag &
2444 local num_failovers=0
2445 local start_ts=$SECONDS
2446 while [ $elapsed -lt $duration ]; do
2450 num_failovers=$((num_failovers + 1))
2451 log "$TESTNAME failing OST $num_failovers times"
2454 elapsed=$((SECONDS - start_ts))
2461 run_test 70f "OSS O_DIRECT recovery with $CLIENTCOUNT clients"
2465 kill -9 $mkdir_71a_pid
2468 random_double_fail_mdt() {
2471 local monitor_pid=$3
2473 local start_ts=$(date +%s)
2474 local num_failovers=0
2478 elapsed=$(($(date +%s) - start_ts))
2479 while [ $elapsed -lt $duration ]; do
2480 fail_index=$((RANDOM%max_index + 1))
2481 if [ $fail_index -eq $max_index ]; then
2484 second_index=$((fail_index + 1))
2486 kill -0 $monitor_pid ||
2487 error "$monitor_pid stopped"
2489 replay_barrier mds$fail_index
2490 replay_barrier mds$second_index
2492 # Increment the number of failovers
2493 num_failovers=$((num_failovers+1))
2494 log "fail mds$fail_index mds$second_index $num_failovers times"
2495 fail mds${fail_index},mds${second_index}
2496 elapsed=$(($(date +%s) - start_ts))
2501 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2502 local clients=${CLIENTS:-$HOSTNAME}
2505 zconf_mount_clients $clients $MOUNT
2508 [ "$SLOW" = "no" ] && duration=180
2509 # set duration to 900 because it takes some time to boot node
2510 [ "$FAILURE_MODE" = HARD ] && duration=900
2512 mkdir_on_mdt0 $DIR/$tdir
2515 local start_ts=$(date +%s)
2517 trap cleanup_71a EXIT
2520 $LFS mkdir -i0 -c2 $DIR/$tdir/test
2521 rmdir $DIR/$tdir/test
2525 echo "Started $mkdir_71a_pid"
2527 random_double_fail_mdt 2 $duration $mkdir_71a_pid
2528 kill -0 $mkdir_71a_pid || error "mkdir/rmdir $mkdir_71a_pid stopped"
2533 run_test 71a "mkdir/rmdir striped dir with 2 mdts recovery"
2536 multiop_bg_pause $DIR/$tfile O_tSc ||
2537 error "multiop_bg_pause $DIR/$tfile failed"
2541 replay_barrier $SINGLEMDS
2542 #define OBD_FAIL_LDLM_ENQUEUE_NET 0x302
2543 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000302"
2546 wait $pid || error "multiop pid failed"
2547 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
2550 run_test 73a "open(O_CREAT), unlink, replay, reconnect before open replay, close"
2553 multiop_bg_pause $DIR/$tfile O_tSc ||
2554 error "multiop_bg_pause $DIR/$tfile failed"
2558 replay_barrier $SINGLEMDS
2559 #define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
2560 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000157"
2563 wait $pid || error "multiop pid failed"
2564 [ -e $DIR/$tfile ] && error "file $DIR/$tfile should not exist"
2567 run_test 73b "open(O_CREAT), unlink, replay, reconnect at open_replay reply, close"
2571 local clients=${CLIENTS:-$HOSTNAME}
2573 zconf_umount_clients $clients $MOUNT
2575 facet_failover $SINGLEMDS
2576 zconf_mount_clients $clients $MOUNT
2578 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
2579 rm $DIR/$tfile || error "rm $DIR/$tfile failed"
2580 clients_up || error "client evicted: $?"
2583 run_test 74 "Ensure applications don't fail waiting for OST recovery"
2585 remote_dir_check_80() {
2590 diridx=$($LFS getstripe -m $remote_dir) ||
2591 error "$LFS getstripe -m $remote_dir failed"
2592 [ $diridx -eq $mdtidx ] || error "$diridx != $mdtidx"
2594 createmany -o $remote_dir/f-%d 20 || error "creation failed"
2595 fileidx=$($LFS getstripe -m $remote_dir/f-1) ||
2596 error "$LFS getstripe -m $remote_dir/f-1 failed"
2597 [ $fileidx -eq $mdtidx ] || error "$fileidx != $mdtidx"
2603 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2604 ([ $FAILURE_MODE == "HARD" ] &&
2605 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2606 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2610 local remote_dir=$DIR/$tdir/remote_dir
2612 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2613 #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2614 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2615 $LFS mkdir -i $MDTIDX $remote_dir &
2621 wait $CLIENT_PID || error "remote creation failed"
2623 remote_dir_check_80 || error "remote dir check failed"
2624 rm -rf $DIR/$tdir || error "rmdir failed"
2628 run_test 80a "DNE: create remote dir, drop update rep from MDT0, fail MDT0"
2631 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2632 ([ $FAILURE_MODE == "HARD" ] &&
2633 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2634 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2638 local remote_dir=$DIR/$tdir/remote_dir
2640 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2641 #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2642 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2643 $LFS mkdir -i $MDTIDX $remote_dir &
2648 fail mds$((MDTIDX + 1))
2650 wait $CLIENT_PID || error "remote creation failed"
2652 remote_dir_check_80 || error "remote dir check failed"
2653 rm -rf $DIR/$tdir || error "rmdir failed"
2657 run_test 80b "DNE: create remote dir, drop update rep from MDT0, fail MDT1"
2660 [[ "$mds1_FSTYPE" = zfs ]] &&
2661 [[ $MDS1_VERSION -lt $(version_code 2.12.51) ]] &&
2662 skip "requires LU-10143 fix on MDS"
2663 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
2664 ([ $FAILURE_MODE == "HARD" ] &&
2665 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2666 skip "MDTs needs to be on diff hosts for HARD fail mode"
2669 local remote_dir=$DIR/$tdir/remote_dir
2671 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2672 #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2673 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2674 $LFS mkdir -i $MDTIDX $remote_dir &
2680 fail mds$((MDTIDX + 1))
2682 wait $CLIENT_PID || error "remote creation failed"
2684 remote_dir_check_80 || error "remote dir check failed"
2685 rm -rf $DIR/$tdir || error "rmdir failed"
2689 run_test 80c "DNE: create remote dir, drop update rep from MDT1, fail MDT[0,1]"
2692 [[ "$mds1_FSTYPE" = zfs ]] &&
2693 [[ $MDS1_VERSION -lt $(version_code 2.12.51) ]] &&
2694 skip "requires LU-10143 fix on MDS"
2695 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
2697 local remote_dir=$DIR/$tdir/remote_dir
2699 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2700 #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2701 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2702 $LFS mkdir -i $MDTIDX $remote_dir &
2705 # sleep 3 seconds to make sure MDTs are failed after
2706 # lfs mkdir -i has finished on all of MDTs.
2711 fail mds${MDTIDX},mds$((MDTIDX + 1))
2713 wait $CLIENT_PID || error "remote creation failed"
2715 remote_dir_check_80 || error "remote dir check failed"
2716 rm -rf $DIR/$tdir || error "rmdir failed"
2720 run_test 80d "DNE: create remote dir, drop update rep from MDT1, fail 2 MDTs"
2723 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2724 ([ $FAILURE_MODE == "HARD" ] &&
2725 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2726 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2730 local remote_dir=$DIR/$tdir/remote_dir
2732 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2733 # OBD_FAIL_MDS_REINT_NET_REP 0x119
2734 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
2735 $LFS mkdir -i $MDTIDX $remote_dir &
2738 # sleep 3 seconds to make sure MDTs are failed after
2739 # lfs mkdir -i has finished on all of MDTs.
2745 wait $CLIENT_PID || error "remote creation failed"
2747 remote_dir_check_80 || error "remote dir check failed"
2748 rm -rf $DIR/$tdir || error "rmdir failed"
2752 run_test 80e "DNE: create remote dir, drop MDT1 rep, fail MDT0"
2755 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2756 ([ $FAILURE_MODE == "HARD" ] &&
2757 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2758 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2761 local remote_dir=$DIR/$tdir/remote_dir
2763 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2764 # OBD_FAIL_MDS_REINT_NET_REP 0x119
2765 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
2766 $LFS mkdir -i $MDTIDX $remote_dir &
2770 fail mds$((MDTIDX + 1))
2772 wait $CLIENT_PID || error "remote creation failed"
2774 remote_dir_check_80 || error "remote dir check failed"
2775 rm -rf $DIR/$tdir || error "rmdir failed"
2779 run_test 80f "DNE: create remote dir, drop MDT1 rep, fail MDT1"
2782 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2783 ([ $FAILURE_MODE == "HARD" ] &&
2784 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2785 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2789 local remote_dir=$DIR/$tdir/remote_dir
2791 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2792 # OBD_FAIL_MDS_REINT_NET_REP 0x119
2793 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
2794 $LFS mkdir -i $MDTIDX $remote_dir &
2797 # sleep 3 seconds to make sure MDTs are failed after
2798 # lfs mkdir -i has finished on all of MDTs.
2804 fail mds$((MDTIDX + 1))
2806 wait $CLIENT_PID || error "remote creation failed"
2808 remote_dir_check_80 || error "remote dir check failed"
2809 rm -rf $DIR/$tdir || error "rmdir failed"
2813 run_test 80g "DNE: create remote dir, drop MDT1 rep, fail MDT0, then MDT1"
2816 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2818 local remote_dir=$DIR/$tdir/remote_dir
2820 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2821 # OBD_FAIL_MDS_REINT_NET_REP 0x119
2822 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
2823 $LFS mkdir -i $MDTIDX $remote_dir &
2826 # sleep 3 seconds to make sure MDTs are failed after
2827 # lfs mkdir -i has finished on all of MDTs.
2832 fail mds${MDTIDX},mds$((MDTIDX + 1))
2834 wait $CLIENT_PID || error "remote dir creation failed"
2836 remote_dir_check_80 || error "remote dir check failed"
2837 rm -rf $DIR/$tdir || error "rmdir failed"
2841 run_test 80h "DNE: create remote dir, drop MDT1 rep, fail 2 MDTs"
2844 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2845 ([ $FAILURE_MODE == "HARD" ] &&
2846 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2847 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2851 local remote_dir=$DIR/$tdir/remote_dir
2853 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2854 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
2856 touch $remote_dir || error "touch $remote_dir failed"
2857 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2858 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2863 fail mds$((MDTIDX + 1))
2865 wait $CLIENT_PID || error "rm remote dir failed"
2867 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
2869 rm -rf $DIR/$tdir || error "rmdir failed"
2873 run_test 81a "DNE: unlink remote dir, drop MDT0 update rep, fail MDT1"
2876 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2877 ([ $FAILURE_MODE == "HARD" ] &&
2878 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2879 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2882 local remote_dir=$DIR/$tdir/remote_dir
2884 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2885 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
2887 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2888 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2895 wait $CLIENT_PID || error "rm remote dir failed"
2897 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
2899 rm -rf $DIR/$tdir || error "rmdir failed"
2903 run_test 81b "DNE: unlink remote dir, drop MDT0 update reply, fail MDT0"
2906 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2907 ([ $FAILURE_MODE == "HARD" ] &&
2908 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2909 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2913 local remote_dir=$DIR/$tdir/remote_dir
2915 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2916 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
2918 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2919 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2926 fail mds$((MDTIDX + 1))
2928 wait $CLIENT_PID || error "rm remote dir failed"
2930 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
2932 rm -rf $DIR/$tdir || error "rmdir failed"
2936 run_test 81c "DNE: unlink remote dir, drop MDT0 update reply, fail MDT0,MDT1"
2939 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2941 local remote_dir=$DIR/$tdir/remote_dir
2943 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2944 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
2946 # OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
2947 do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
2953 fail mds${MDTIDX},mds$((MDTIDX + 1))
2955 wait $CLIENT_PID || error "rm remote dir failed"
2957 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
2959 rm -rf $DIR/$tdir || error "rmdir failed"
2963 run_test 81d "DNE: unlink remote dir, drop MDT0 update reply, fail 2 MDTs"
2966 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2967 ([ $FAILURE_MODE == "HARD" ] &&
2968 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
2969 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
2973 local remote_dir=$DIR/$tdir/remote_dir
2975 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2976 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
2978 # OBD_FAIL_MDS_REINT_NET_REP 0x119
2979 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
2982 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
2987 wait $CLIENT_PID || error "rm remote dir failed"
2989 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
2991 rm -rf $DIR/$tdir || error "rmdir failed"
2995 run_test 81e "DNE: unlink remote dir, drop MDT1 req reply, fail MDT0"
2998 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
2999 ([ $FAILURE_MODE == "HARD" ] &&
3000 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3001 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3005 local remote_dir=$DIR/$tdir/remote_dir
3007 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3008 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
3010 # OBD_FAIL_MDS_REINT_NET_REP 0x119
3011 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
3016 fail mds$((MDTIDX + 1))
3018 wait $CLIENT_PID || error "rm remote dir failed"
3020 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
3022 rm -rf $DIR/$tdir || error "rmdir failed"
3026 run_test 81f "DNE: unlink remote dir, drop MDT1 req reply, fail MDT1"
3029 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3030 ([ $FAILURE_MODE == "HARD" ] &&
3031 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3032 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3036 local remote_dir=$DIR/$tdir/remote_dir
3038 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3039 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
3041 # OBD_FAIL_MDS_REINT_NET_REP 0x119
3042 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
3049 fail mds$((MDTIDX + 1))
3051 wait $CLIENT_PID || error "rm remote dir failed"
3053 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
3055 rm -rf $DIR/$tdir || error "rmdir failed"
3059 run_test 81g "DNE: unlink remote dir, drop req reply, fail M0, then M1"
3062 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3064 local remote_dir=$DIR/$tdir/remote_dir
3066 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3067 $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
3069 # OBD_FAIL_MDS_REINT_NET_REP 0x119
3070 do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
3076 fail mds${MDTIDX},mds$((MDTIDX + 1))
3078 wait $CLIENT_PID || error "rm remote dir failed"
3080 stat $remote_dir &>/dev/null && error "$remote_dir still exist!"
3082 rm -rf $DIR/$tdir || error "rmdir failed"
3086 run_test 81h "DNE: unlink remote dir, drop request reply, fail 2 MDTs"
3089 #define OBD_FAIL_MDS_OPEN_WAIT_CREATE 0x144
3090 do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000144"
3091 createmany -o $DIR/$tfile- 1 &
3095 client_up || client_up || true # reconnect
3097 run_test 84a "stale open during export disconnect"
3099 test_85a() { #bug 16774
3100 lctl set_param -n ldlm.cancel_unused_locks_before_replay "1"
3102 for i in $(seq 100); do
3103 echo "tag-$i" > $DIR/$tfile-$i
3104 grep -q "tag-$i" $DIR/$tfile-$i || error "f2-$i"
3107 lov_id=$(lctl dl | grep "clilov")
3108 addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $NF}')
3109 count=$(lctl get_param -n \
3110 ldlm.namespaces.*MDT0000*$addr.lock_unused_count)
3111 echo "before recovery: unused locks count = $count"
3115 count2=$(lctl get_param -n \
3116 ldlm.namespaces.*MDT0000*$addr.lock_unused_count)
3117 echo "after recovery: unused locks count = $count2"
3119 if [ $count2 -ge $count ]; then
3120 error "unused locks are not canceled"
3123 run_test 85a "check the cancellation of unused locks during recovery(IBITS)"
3125 test_85b() { #bug 16774
3129 lctl set_param -n ldlm.cancel_unused_locks_before_replay "1"
3131 $LFS setstripe -c 1 -i 0 $DIR/$tdir
3133 for i in $(seq 100); do
3134 dd if=/dev/urandom of=$DIR/$tdir/$tfile-$i bs=4096 \
3135 count=32 >/dev/null 2>&1
3138 cancel_lru_locks osc
3140 for i in $(seq 100); do
3141 dd if=$DIR/$tdir/$tfile-$i of=/dev/null bs=4096 \
3142 count=32 >/dev/null 2>&1
3145 lov_id=$(lctl dl | grep "clilov")
3146 addr=$(echo $lov_id | awk '{print $4}' | awk -F '-' '{print $NF}')
3147 count=$(lctl get_param -n \
3148 ldlm.namespaces.*OST0000*$addr.lock_unused_count)
3149 echo "before recovery: unused locks count = $count"
3150 [ $count -ne 0 ] || error "unused locks ($count) should be zero"
3154 count2=$(lctl get_param \
3155 -n ldlm.namespaces.*OST0000*$addr.lock_unused_count)
3156 echo "after recovery: unused locks count = $count2"
3158 if [ $count2 -ge $count ]; then
3159 error "unused locks are not canceled"
3164 run_test 85b "check the cancellation of unused locks during recovery(EXTENT)"
3167 local clients=${CLIENTS:-$HOSTNAME}
3169 zconf_umount_clients $clients $MOUNT
3170 do_facet $SINGLEMDS lctl set_param mdt.${FSNAME}-MDT*.exports.clear=0
3171 remount_facet $SINGLEMDS
3172 zconf_mount_clients $clients $MOUNT
3174 run_test 86 "umount server after clear nid_stats should not hit LBUG"
3177 do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.sync_journal 0"
3180 $LFS setstripe -i 0 -c 1 $DIR/$tfile
3181 dd if=/dev/urandom of=$DIR/$tfile bs=1024k count=8 ||
3182 error "dd to $DIR/$tfile failed"
3183 cksum=$(md5sum $DIR/$tfile | awk '{print $1}')
3184 cancel_lru_locks osc
3186 dd if=$DIR/$tfile of=/dev/null bs=1024k count=8 || error "Cannot read"
3187 cksum2=$(md5sum $DIR/$tfile | awk '{print $1}')
3188 if [ $cksum != $cksum2 ] ; then
3189 error "New checksum $cksum2 does not match original $cksum"
3192 run_test 87a "write replay"
3195 do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.sync_journal 0"
3198 $LFS setstripe -i 0 -c 1 $DIR/$tfile
3199 dd if=/dev/urandom of=$DIR/$tfile bs=1024k count=8 ||
3200 error "dd to $DIR/$tfile failed"
3201 sleep 1 # Give it a chance to flush dirty data
3202 echo TESTTEST | dd of=$DIR/$tfile bs=1 count=8 seek=64
3203 cksum=$(md5sum $DIR/$tfile | awk '{print $1}')
3204 cancel_lru_locks osc
3206 dd if=$DIR/$tfile of=/dev/null bs=1024k count=8 || error "Cannot read"
3207 cksum2=$(md5sum $DIR/$tfile | awk '{print $1}')
3208 if [ $cksum != $cksum2 ] ; then
3209 error "New checksum $cksum2 does not match original $cksum"
3212 run_test 87b "write replay with changed data (checksum resend)"
3214 test_88() { #bug 17485
3215 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3216 mkdir -p $TMP/$tdir || error "mkdir $TMP/$tdir failed"
3218 $LFS setstripe -i 0 -c 1 $DIR/$tdir || error "$LFS setstripe failed"
3221 replay_barrier $SINGLEMDS
3223 # exhaust precreations on ost1
3224 local OST=$(ostname_from_index 0)
3225 local mdtosc=$(get_mdtosc_proc_path $SINGLEMDS $OST)
3226 local last_id=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_last_id)
3227 local next_id=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_next_id)
3228 echo "before test: last_id = $last_id, next_id = $next_id"
3230 echo "Creating to objid $last_id on ost $OST..."
3231 createmany -o $DIR/$tdir/f-%d $next_id $((last_id - next_id + 2)) ||
3232 error "createmany create files to last_id failed"
3234 #create some files to use some uncommitted objids
3235 last_id=$(($last_id + 1))
3236 createmany -o $DIR/$tdir/f-%d $last_id 8 ||
3237 error "createmany create files with uncommitted objids failed"
3239 last_id2=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_last_id)
3240 next_id2=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_next_id)
3241 echo "before recovery: last_id = $last_id2, next_id = $next_id2"
3243 # if test uses shutdown_facet && reboot_facet instead of facet_failover ()
3244 # it has to take care about the affected facets, bug20407
3245 local affected_mds1=$(affected_facets mds1)
3246 local affected_ost1=$(affected_facets ost1)
3248 shutdown_facet $SINGLEMDS
3251 reboot_facet $SINGLEMDS
3252 change_active $affected_mds1
3253 wait_for_facet $affected_mds1
3254 mount_facets $affected_mds1 || error "Restart of mds failed"
3257 change_active $affected_ost1
3258 wait_for_facet $affected_ost1
3259 mount_facets $affected_ost1 || error "Restart of ost1 failed"
3263 last_id2=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_last_id)
3264 next_id2=$(do_facet $SINGLEMDS lctl get_param -n osp.$mdtosc.prealloc_next_id)
3265 echo "after recovery: last_id = $last_id2, next_id = $next_id2"
3267 # create new files, which should use new objids, and ensure the orphan
3268 # cleanup phase for ost1 is completed at the same time
3269 for i in $(seq 8); do
3270 file_id=$(($last_id + 10 + $i))
3271 dd if=/dev/urandom of=$DIR/$tdir/f-$file_id bs=4096 count=128
3274 # if the objids were not recreated, then "ls" will fail with -ENOENT
3275 ls -l $DIR/$tdir/* || error "can't get the status of precreated files"
3278 # write into previously created files
3279 for i in $(seq 8); do
3280 file_id=$(($last_id + $i))
3281 dd if=/dev/urandom of=$DIR/$tdir/f-$file_id bs=4096 count=128
3282 cp -f $DIR/$tdir/f-$file_id $TMP/$tdir/
3285 # compare the content
3286 for i in $(seq 8); do
3287 file_id=$(($last_id + $i))
3288 cmp $TMP/$tdir/f-$file_id $DIR/$tdir/f-$file_id ||
3289 error "the content of file is modified!"
3294 run_test 88 "MDS should not assign same objid to different files "
3296 function calc_osc_kbytes_used() {
3297 local kbtotal=$(calc_osc_kbytes kbytestotal)
3298 local kbfree=$(calc_osc_kbytes kbytesfree)
3299 echo $((kbtotal-kbfree))
3303 cancel_lru_locks osc
3304 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3305 rm -f $DIR/$tdir/$tfile
3306 wait_mds_ost_sync || error "initial MDS-OST sync timed out"
3307 wait_delete_completed || error "initial wait delete timed out"
3308 local blocks1=$(calc_osc_kbytes_used)
3309 local write_size=$(fs_log_size)
3311 $LFS setstripe -i 0 -c 1 $DIR/$tdir/$tfile
3312 [ $write_size -lt 1024 ] && write_size=1024
3313 dd if=/dev/zero bs=${write_size}k count=10 of=$DIR/$tdir/$tfile
3316 facet_failover $SINGLEMDS
3317 rm $DIR/$tdir/$tfile
3320 zconf_mount $(hostname) $MOUNT || error "mount fails"
3321 client_up || error "client_up failed"
3323 # wait for the remounted client to connect to ost1
3324 local target=$(get_osc_import_name client ost1)
3325 wait_import_state "FULL" "osc.${target}.ost_server_uuid" \
3326 $(max_recovery_time)
3328 wait_mds_ost_sync || error "MDS-OST sync timed out"
3329 wait_delete_completed || error "wait delete timed out"
3330 local blocks2=$(calc_osc_kbytes_used)
3332 [ $((blocks2 - blocks1)) -le $(fs_log_size) ] ||
3333 error $((blocks2 - blocks1)) blocks leaked
3335 run_test 89 "no disk space leak on late ost connection"
3342 change_active $facet
3343 wait_for_facet $facet
3344 mount_facet $facet || error "Restart of $facet failed"
3348 test_90() { # bug 19494
3349 local dir=$DIR/$tdir
3350 local ostfail=$(get_random_entry $(get_facets OST))
3352 if [[ $FAILURE_MODE = HARD ]]; then
3353 local affected=$(affected_facets $ostfail);
3355 [[ "$affected" == $ostfail ]] ||
3356 skip "cannot use FAILURE_MODE=$FAILURE_MODE, affected: $affected"
3358 # ensure all OSTs are active to allow allocations
3361 mkdir $dir || error "mkdir $dir failed"
3363 echo "Create the files"
3365 # file "f${index}" striped over 1 OST
3366 # file "all" striped over all OSTs
3368 $LFS setstripe -c $OSTCOUNT $dir/all ||
3369 error "setstripe failed to create $dir/all"
3371 for ((i = 0; i < $OSTCOUNT; i++)); do
3374 $LFS setstripe -i $i -c 1 $f ||
3375 error "$LFS setstripe failed to create $f"
3377 # confirm setstripe actually created stripe on requested OST
3378 local uuid=$(ostuuid_from_index $i)
3380 for file in f$i all; do
3381 local found=$($LFS find --obd $uuid --name $file $dir)
3383 if [[ $dir/$file != $found ]]; then
3384 $LFS getstripe $dir/$file
3385 error "wrong stripe: $file, uuid: $uuid"
3390 # Before failing an OST, get its obd name and index
3391 local varsvc=${ostfail}_svc
3392 local obd=$(do_facet $ostfail lctl get_param \
3393 -n obdfilter.${!varsvc}.uuid)
3394 local index=$(($(facet_number $ostfail) - 1))
3396 echo "Fail $ostfail $obd, display the list of affected files"
3397 shutdown_facet $ostfail || error "shutdown_facet $ostfail failed"
3399 trap "cleanup_90 $ostfail" EXIT INT
3400 echo "General Query: lfs find $dir"
3401 local list=$($LFS find $dir)
3403 for (( i=0; i<$OSTCOUNT; i++ )); do
3404 list_member "$list" $dir/f$i ||
3405 error_noexit "lfs find $dir: no file f$i"
3407 list_member "$list" $dir/all ||
3408 error_noexit "lfs find $dir: no file all"
3410 # focus on the missing OST,
3411 # we expect to see only two files affected: "f$(index)" and "all"
3413 echo "Querying files on shutdown $ostfail: lfs find --obd $obd"
3414 list=$($LFS find --obd $obd $dir)
3416 for file in all f$index; do
3417 list_member "$list" $dir/$file ||
3418 error_noexit "lfs find does not report the affected $obd for $file"
3421 [[ $(echo $list | wc -w) -eq 2 ]] ||
3422 error_noexit "lfs find reports the wrong list of affected files ${#list[@]}"
3424 echo "Check getstripe: $LFS getstripe -r --obd $obd"
3425 list=$($LFS getstripe -r --obd $obd $dir)
3427 for file in all f$index; do
3428 echo "$list" | grep $dir/$file ||
3429 error_noexit "lfs getsripe does not report the affected $obd for $file"
3434 run_test 90 "lfs find identifies the missing striped file segments"
3437 [[ "$MDS1_VERSION" -ge $(version_code 2.6.90) ]] ||
3438 [[ "$MDS1_VERSION" -ge $(version_code 2.5.4) &&
3439 "$MDS1_VERSION" -lt $(version_code 2.5.50) ]] ||
3440 skip "Need MDS version 2.5.4+ or 2.6.90+"
3442 cancel_lru_locks osc
3444 $LFS setstripe -i 0 -c 1 $DIR/$tfile ||
3445 error "$LFS setstripe $DIR/$tfile failed"
3446 dd if=/dev/zero of=$DIR/$tfile bs=1024 count=1 ||
3447 error "dd to $DIR/$tfile failed"
3448 #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
3449 # We need to emulate a state that OST is waiting for other clients
3450 # not completing the recovery. Final ping is queued, but reply will be
3451 # sent on the recovery completion. It is done by sleep before
3452 # processing final pings
3453 do_facet ost1 "$LCTL set_param fail_val=40"
3454 do_facet ost1 "$LCTL set_param fail_loc=0x715"
3457 run_test 93a "replay + reconnect"
3460 [[ "$MDS1_VERSION" -ge $(version_code 2.7.90) ]] ||
3461 skip "Need MDS version 2.7.90+"
3463 cancel_lru_locks mdc
3465 createmany -o $DIR/$tfile 20 ||
3466 error "createmany -o $DIR/$tfile failed"
3468 #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715
3469 # We need to emulate a state that MDT is waiting for other clients
3470 # not completing the recovery. Final ping is queued, but reply will be
3471 # sent on the recovery completion. It is done by sleep before
3472 # processing final pings
3473 do_facet mds1 "$LCTL set_param fail_val=80"
3474 do_facet mds1 "$LCTL set_param fail_loc=0x715"
3477 run_test 93b "replay + reconnect on mds"
3479 striped_dir_check_100() {
3480 local striped_dir=$DIR/$tdir/striped_dir
3481 local stripe_count=$($LFS getdirstripe -c $striped_dir)
3483 $LFS getdirstripe $striped_dir
3484 [ $stripe_count -eq 2 ] || error "$stripe_count != 2"
3486 createmany -o $striped_dir/f-%d 20 ||
3487 error "creation failed under striped dir"
3491 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3492 ([ $FAILURE_MODE == "HARD" ] &&
3493 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3494 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3497 local striped_dir=$DIR/$tdir/striped_dir
3500 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3502 #To make sure MDT1 and MDT0 are connected
3503 #otherwise it may create single stripe dir here
3504 $LFS setdirstripe -i1 $DIR/$tdir/remote_dir
3506 #define OBD_FAIL_OUT_UPDATE_NET_REP 0x1701
3507 do_facet mds$((MDTIDX+1)) lctl set_param fail_loc=0x1701
3508 $LFS setdirstripe -i0 -c2 $striped_dir &
3511 fail mds$((MDTIDX + 1))
3513 wait $CLIENT_PID || error "striped dir creation failed"
3515 striped_dir_check_100 || error "striped dir check failed"
3516 rm -rf $DIR/$tdir || error "rmdir failed"
3518 run_test 100a "DNE: create striped dir, drop update rep from MDT1, fail MDT1"
3521 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3522 ([ $FAILURE_MODE == "HARD" ] &&
3523 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3524 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3527 local striped_dir=$DIR/$tdir/striped_dir
3530 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3532 #To make sure MDT1 and MDT0 are connected
3533 #otherwise it may create single stripe dir here
3534 $LFS setdirstripe -i1 $DIR/$tdir/remote_dir
3536 # OBD_FAIL_MDS_REINT_NET_REP 0x119
3537 do_facet mds$MDTIDX lctl set_param fail_loc=0x119
3538 $LFS mkdir -i0 -c2 $striped_dir &
3543 wait $CLIENT_PID || error "striped dir creation failed"
3545 striped_dir_check_100 || error "striped dir check failed"
3546 rm -rf $DIR/$tdir || error "rmdir failed"
3548 run_test 100b "DNE: create striped dir, fail MDT0"
3551 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3552 ([ $FAILURE_MODE == "HARD" ] &&
3553 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3554 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3557 local striped_dir=$DIR/$tdir/striped_dir
3559 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3561 #To make sure MDT1 and MDT0 are connected
3562 #otherwise it may create single stripe dir here
3563 $LFS setdirstripe -i1 $DIR/$tdir/remote_dir
3566 $LFS mkdir -i1 -c2 $striped_dir
3568 fail_abort mds2 abort_recov_mdt
3570 createmany -o $striped_dir/f-%d 20 &&
3571 error "createmany -o $DIR/$tfile should fail"
3574 striped_dir_check_100 || error "striped dir check failed"
3575 rm -rf $DIR/$tdir || error "rmdir failed"
3577 run_test 100c "DNE: create striped dir, fail MDT0"
3579 test_101() { #LU-5648
3580 mkdir -p $DIR/$tdir/d1
3581 mkdir -p $DIR/$tdir/d2
3582 touch $DIR/$tdir/file0
3585 replay_barrier $SINGLEMDS
3586 for i in $(seq $num) ; do
3587 echo test$i > $DIR/$tdir/d1/file$i
3590 fail_abort $SINGLEMDS
3591 for i in $(seq $num) ; do
3592 touch $DIR/$tdir/d2/file$i
3593 test -s $DIR/$tdir/d2/file$i &&
3594 ls -al $DIR/$tdir/d2/file$i && error "file$i's size > 0"
3599 run_test 101 "Shouldn't reassign precreated objs to other files after recovery"
3608 [[ $(lctl get_param mdc.*.import |
3609 grep "connect_flags:.*multi_mod_rpc") ]] ||
3610 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
3612 $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3613 idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
3614 facet="mds$((0x$idx + 1))"
3616 # get current value of max_mod_rcps_in_flight
3617 num=$($LCTL get_param -n \
3618 mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
3619 # set default value if client does not support multi mod RPCs
3620 [ -z "$num" ] && num=1
3622 echo "creating $num files ..."
3624 for i in $(seq $num); do
3625 touch $DIR/$tdir/file-$i
3628 # drop request on MDT to force resend
3629 #define OBD_FAIL_MDS_REINT_MULTI_NET 0x159
3630 do_facet $facet "$LCTL set_param fail_loc=0x159"
3631 echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
3632 for i in $(seq $num); do
3633 chmod 0600 $DIR/$tdir/file-$i &
3637 do_facet $facet "$LCTL set_param fail_loc=0"
3638 for pid in $pids; do
3639 wait $pid || error "chmod failed"
3641 echo "done ($(date +%H:%M:%S))"
3643 # check chmod succeed
3644 for i in $(seq $num); do
3645 checkstat -vp 0600 $DIR/$tdir/file-$i
3650 run_test 102a "check resend (request lost) with multiple modify RPCs in flight"
3659 [[ $(lctl get_param mdc.*.import |
3660 grep "connect_flags:.*multi_mod_rpc") ]] ||
3661 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
3663 $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3664 idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
3665 facet="mds$((0x$idx + 1))"
3667 # get current value of max_mod_rcps_in_flight
3668 num=$($LCTL get_param -n \
3669 mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
3670 # set default value if client does not support multi mod RPCs
3671 [ -z "$num" ] && num=1
3673 echo "creating $num files ..."
3675 for i in $(seq $num); do
3676 touch $DIR/$tdir/file-$i
3679 # drop reply on MDT to force reconstruction
3680 #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
3681 do_facet $facet "$LCTL set_param fail_loc=0x15a"
3682 echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
3683 for i in $(seq $num); do
3684 chmod 0600 $DIR/$tdir/file-$i &
3688 do_facet $facet "$LCTL set_param fail_loc=0"
3689 for pid in $pids; do
3690 wait $pid || error "chmod failed"
3692 echo "done ($(date +%H:%M:%S))"
3694 # check chmod succeed
3695 for i in $(seq $num); do
3696 checkstat -vp 0600 $DIR/$tdir/file-$i
3701 run_test 102b "check resend (reply lost) with multiple modify RPCs in flight"
3710 [[ $(lctl get_param mdc.*.import |
3711 grep "connect_flags:.*multi_mod_rpc") ]] ||
3712 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
3714 $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3715 idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
3716 facet="mds$((0x$idx + 1))"
3718 # get current value of max_mod_rcps_in_flight
3719 num=$($LCTL get_param -n \
3720 mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
3721 # set default value if client does not support multi mod RPCs
3722 [ -z "$num" ] && num=1
3724 echo "creating $num files ..."
3726 for i in $(seq $num); do
3727 touch $DIR/$tdir/file-$i
3730 replay_barrier $facet
3733 #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
3734 do_facet $facet "$LCTL set_param fail_loc=0x15a"
3735 echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
3736 for i in $(seq $num); do
3737 chmod 0600 $DIR/$tdir/file-$i &
3741 do_facet $facet "$LCTL set_param fail_loc=0"
3746 for pid in $pids; do
3747 wait $pid || error "chmod failed"
3749 echo "done ($(date +%H:%M:%S))"
3751 # check chmod succeed
3752 for i in $(seq $num); do
3753 checkstat -vp 0600 $DIR/$tdir/file-$i
3758 run_test 102c "check replay w/o reconstruction with multiple mod RPCs in flight"
3767 [[ $(lctl get_param mdc.*.import |
3768 grep "connect_flags:.*multi_mod_rpc") ]] ||
3769 { skip "Need MDC with 'multi_mod_rpcs' feature"; return 0; }
3771 $LFS mkdir -c1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3772 idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/$tdir))
3773 facet="mds$((0x$idx + 1))"
3775 # get current value of max_mod_rcps_in_flight
3776 num=$($LCTL get_param -n \
3777 mdc.$FSNAME-MDT$idx-mdc-*.max_mod_rpcs_in_flight)
3778 # set default value if client does not support multi mod RPCs
3779 [ -z "$num" ] && num=1
3781 echo "creating $num files ..."
3783 for i in $(seq $num); do
3784 touch $DIR/$tdir/file-$i
3788 #define OBD_FAIL_MDS_REINT_MULTI_NET_REP 0x15a
3789 do_facet $facet "$LCTL set_param fail_loc=0x15a"
3790 echo "launch $num chmod in parallel ($(date +%H:%M:%S)) ..."
3791 for i in $(seq $num); do
3792 chmod 0600 $DIR/$tdir/file-$i &
3797 # write MDT transactions to disk
3798 do_facet $facet "sync; sync; sync"
3800 do_facet $facet "$LCTL set_param fail_loc=0"
3805 for pid in $pids; do
3806 wait $pid || error "chmod failed"
3808 echo "done ($(date +%H:%M:%S))"
3810 # check chmod succeed
3811 for i in $(seq $num); do
3812 checkstat -vp 0600 $DIR/$tdir/file-$i
3817 run_test 102d "check replay & reconstruction with multiple mod RPCs in flight"
3820 remote_mds_nodsh && skip "remote MDS with nodsh"
3821 [[ "$MDS1_VERSION" -gt $(version_code 2.8.54) ]] ||
3822 skip "Need MDS version 2.8.54+"
3824 #define OBD_FAIL_MDS_TRACK_OVERFLOW 0x162
3825 do_facet mds1 $LCTL set_param fail_loc=0x80000162
3828 createmany -o $DIR/$tdir/t- 30 ||
3829 error "create files on remote directory failed"
3831 rm -rf $DIR/$tdir/t-*
3833 #MDS should crash with tr->otr_next_id overflow
3836 run_test 103 "Check otr_next_id overflow"
3839 check_striped_dir_110()
3841 $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
3842 error "create striped dir failed"
3843 local stripe_count=$($LFS getdirstripe -c $DIR/$tdir/striped_dir)
3844 [ $stripe_count -eq $MDSCOUNT ] ||
3845 error "$stripe_count != 2 after recovery"
3849 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3850 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3851 skip "Need MDS version at least 2.7.56"
3853 ([ $FAILURE_MODE == "HARD" ] &&
3854 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3855 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3860 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3863 check_striped_dir_110 || error "check striped_dir failed"
3864 rm -rf $DIR/$tdir || error "rmdir failed"
3868 run_test 110a "DNE: create striped dir, fail MDT1"
3871 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3872 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3873 skip "Need MDS version at least 2.7.56"
3875 ([ $FAILURE_MODE == "HARD" ] &&
3876 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3877 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3882 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3885 zconf_mount $(hostname) $MOUNT
3886 client_up || return 1
3888 check_striped_dir_110 || error "check striped_dir failed"
3890 rm -rf $DIR/$tdir || error "rmdir failed"
3894 run_test 110b "DNE: create striped dir, fail MDT1 and client"
3897 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3898 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3899 skip "Need MDS version at least 2.7.56"
3901 ([ $FAILURE_MODE == "HARD" ] &&
3902 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3903 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3908 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3911 check_striped_dir_110 || error "check striped_dir failed"
3913 rm -rf $DIR/$tdir || error "rmdir failed"
3917 run_test 110c "DNE: create striped dir, fail MDT2"
3920 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3921 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3922 skip "Need MDS version at least 2.7.56"
3924 ([ $FAILURE_MODE == "HARD" ] &&
3925 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3926 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3931 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3934 zconf_mount $(hostname) $MOUNT
3935 client_up || return 1
3937 check_striped_dir_110 || error "check striped_dir failed"
3939 rm -rf $DIR/$tdir || error "rmdir failed"
3943 run_test 110d "DNE: create striped dir, fail MDT2 and client"
3946 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3947 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3948 skip "Need MDS version at least 2.7.56"
3950 ([ $FAILURE_MODE == "HARD" ] &&
3951 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3952 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3957 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3961 zconf_mount $(hostname) $MOUNT
3962 client_up || return 1
3964 check_striped_dir_110 || error "check striped_dir failed"
3966 rm -rf $DIR/$tdir || error "rmdir failed"
3970 run_test 110e "DNE: create striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
3973 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3974 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3975 skip "Need MDS version at least 2.7.56"
3977 ([ $FAILURE_MODE == "HARD" ] &&
3978 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
3979 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
3985 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
3988 check_striped_dir_110 || error "check striped_dir failed"
3990 rm -rf $DIR/$tdir || error "rmdir failed"
3994 run_test 110f "DNE: create striped dir, fail MDT1/MDT2"
3997 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
3998 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
3999 skip "Need MDS version at least 2.7.56"
4001 ([ $FAILURE_MODE == "HARD" ] &&
4002 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4003 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4008 $LFS mkdir -i1 -c$MDSCOUNT $DIR/$tdir/striped_dir
4012 zconf_mount $(hostname) $MOUNT
4013 client_up || return 1
4015 check_striped_dir_110 || error "check striped_dir failed"
4017 rm -rf $DIR/$tdir || error "rmdir failed"
4021 run_test 110g "DNE: create striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
4024 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4025 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4026 skip "Need MDS version at least 2.7.56"
4028 ([ $FAILURE_MODE == "HARD" ] &&
4029 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4030 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4034 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4036 rm -rf $DIR/$tdir/striped_dir
4039 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4040 error "striped dir still exists"
4043 run_test 111a "DNE: unlink striped dir, fail MDT1"
4046 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4047 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4048 skip "Need MDS version at least 2.7.56"
4050 ([ $FAILURE_MODE == "HARD" ] &&
4051 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4052 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4056 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4058 rm -rf $DIR/$tdir/striped_dir
4061 zconf_mount $(hostname) $MOUNT
4062 client_up || return 1
4064 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4065 error "striped dir still exists"
4068 run_test 111b "DNE: unlink striped dir, fail MDT2"
4071 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4072 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4073 skip "Need MDS version at least 2.7.56"
4075 ([ $FAILURE_MODE == "HARD" ] &&
4076 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4077 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4081 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4083 rm -rf $DIR/$tdir/striped_dir
4087 zconf_mount $(hostname) $MOUNT
4088 client_up || return 1
4089 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4090 error "striped dir still exists"
4093 run_test 111c "DNE: unlink striped dir, uncommit on MDT1, fail client/MDT1/MDT2"
4096 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4097 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4098 skip "Need MDS version at least 2.7.56"
4100 ([ $FAILURE_MODE == "HARD" ] &&
4101 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4102 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4106 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4108 rm -rf $DIR/$tdir/striped_dir
4112 zconf_mount $(hostname) $MOUNT
4113 client_up || return 1
4114 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4115 error "striped dir still exists"
4119 run_test 111d "DNE: unlink striped dir, uncommit on MDT2, fail client/MDT1/MDT2"
4122 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4123 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4124 skip "Need MDS version at least 2.7.56"
4126 ([ $FAILURE_MODE == "HARD" ] &&
4127 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4128 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4132 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4134 rm -rf $DIR/$tdir/striped_dir
4137 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4138 error "striped dir still exists"
4141 run_test 111e "DNE: unlink striped dir, uncommit on MDT2, fail MDT1/MDT2"
4144 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4145 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4146 skip "Need MDS version at least 2.7.56"
4148 ([ $FAILURE_MODE == "HARD" ] &&
4149 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4150 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4154 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4156 rm -rf $DIR/$tdir/striped_dir
4159 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4160 error "striped dir still exists"
4163 run_test 111f "DNE: unlink striped dir, uncommit on MDT1, fail MDT1/MDT2"
4166 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4167 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4168 skip "Need MDS version at least 2.7.56"
4170 ([ $FAILURE_MODE == "HARD" ] &&
4171 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4172 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4176 $LFS mkdir -i1 -c2 $DIR/$tdir/striped_dir
4179 rm -rf $DIR/$tdir/striped_dir
4181 $CHECKSTAT -t dir $DIR/$tdir/striped_dir &&
4182 error "striped dir still exists"
4185 run_test 111g "DNE: unlink striped dir, fail MDT1/MDT2"
4187 test_112_rename_prepare() {
4188 mkdir_on_mdt0 $DIR/$tdir
4189 mkdir -p $DIR/$tdir/src_dir
4190 $LFS mkdir -i 1 $DIR/$tdir/src_dir/src_child ||
4191 error "create remote source failed"
4193 touch $DIR/$tdir/src_dir/src_child/a
4195 $LFS mkdir -i 2 $DIR/$tdir/tgt_dir ||
4196 error "create remote target dir failed"
4198 $LFS mkdir -i 3 $DIR/$tdir/tgt_dir/tgt_child ||
4199 error "create remote target child failed"
4204 $CHECKSTAT -t dir $DIR/$tdir/src_dir/src_child &&
4205 error "src_child still exists after rename"
4207 $CHECKSTAT -t file $DIR/$tdir/tgt_dir/tgt_child/a ||
4208 error "missing file(a) after rename"
4212 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4213 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4214 skip "Need MDS version at least 2.7.56"
4216 ([ $FAILURE_MODE == "HARD" ] &&
4217 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4218 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4221 test_112_rename_prepare
4224 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4225 error "rename dir cross MDT failed!"
4229 rm -rf $DIR/$tdir || error "rmdir failed"
4231 run_test 112a "DNE: cross MDT rename, fail MDT1"
4234 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4235 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4236 skip "Need MDS version at least 2.7.56"
4238 ([ $FAILURE_MODE == "HARD" ] &&
4239 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4240 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4243 test_112_rename_prepare
4246 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4247 error "rename dir cross MDT failed!"
4252 rm -rf $DIR/$tdir || error "rmdir failed"
4254 run_test 112b "DNE: cross MDT rename, fail MDT2"
4257 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4258 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4259 skip "Need MDS version at least 2.7.56"
4261 ([ $FAILURE_MODE == "HARD" ] &&
4262 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4263 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4266 test_112_rename_prepare
4269 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4270 error "rename dir cross MDT failed!"
4275 rm -rf $DIR/$tdir || error "rmdir failed"
4277 run_test 112c "DNE: cross MDT rename, fail MDT3"
4280 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4281 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4282 skip "Need MDS version at least 2.7.56"
4284 ([ $FAILURE_MODE == "HARD" ] &&
4285 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4286 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4289 test_112_rename_prepare
4292 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4293 error "rename dir cross MDT failed!"
4298 rm -rf $DIR/$tdir || error "rmdir failed"
4300 run_test 112d "DNE: cross MDT rename, fail MDT4"
4303 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4304 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4305 skip "Need MDS version at least 2.7.56"
4307 ([ $FAILURE_MODE == "HARD" ] &&
4308 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4309 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4312 test_112_rename_prepare
4316 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4317 error "rename dir cross MDT failed!"
4322 rm -rf $DIR/$tdir || error "rmdir failed"
4324 run_test 112e "DNE: cross MDT rename, fail MDT1 and MDT2"
4327 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4328 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4329 skip "Need MDS version at least 2.7.56"
4331 ([ $FAILURE_MODE == "HARD" ] &&
4332 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4333 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4336 test_112_rename_prepare
4340 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4341 error "rename dir cross MDT failed!"
4346 rm -rf $DIR/$tdir || error "rmdir failed"
4348 run_test 112f "DNE: cross MDT rename, fail MDT1 and MDT3"
4351 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4352 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4353 skip "Need MDS version at least 2.7.56"
4355 ([ $FAILURE_MODE == "HARD" ] &&
4356 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4357 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4360 test_112_rename_prepare
4364 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4365 error "rename dir cross MDT failed!"
4370 rm -rf $DIR/$tdir || error "rmdir failed"
4372 run_test 112g "DNE: cross MDT rename, fail MDT1 and MDT4"
4375 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4376 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4377 skip "Need MDS version at least 2.7.56"
4379 ([ $FAILURE_MODE == "HARD" ] &&
4380 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4381 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4384 test_112_rename_prepare
4388 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4389 error "rename dir cross MDT failed!"
4394 rm -rf $DIR/$tdir || error "rmdir failed"
4396 run_test 112h "DNE: cross MDT rename, fail MDT2 and MDT3"
4399 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4400 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4401 skip "Need MDS version at least 2.7.56"
4403 ([ $FAILURE_MODE == "HARD" ] &&
4404 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4405 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4408 test_112_rename_prepare
4412 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4413 error "rename dir cross MDT failed!"
4418 rm -rf $DIR/$tdir || error "rmdir failed"
4420 run_test 112i "DNE: cross MDT rename, fail MDT2 and MDT4"
4423 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4424 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4425 skip "Need MDS version at least 2.7.56"
4427 ([ $FAILURE_MODE == "HARD" ] &&
4428 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4429 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4432 test_112_rename_prepare
4436 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4437 error "rename dir cross MDT failed!"
4442 rm -rf $DIR/$tdir || error "rmdir failed"
4444 run_test 112j "DNE: cross MDT rename, fail MDT3 and MDT4"
4447 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4448 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4449 skip "Need MDS version at least 2.7.56"
4451 ([ $FAILURE_MODE == "HARD" ] &&
4452 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4453 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4456 test_112_rename_prepare
4461 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4462 error "rename dir cross MDT failed!"
4467 rm -rf $DIR/$tdir || error "rmdir failed"
4469 run_test 112k "DNE: cross MDT rename, fail MDT1,MDT2,MDT3"
4472 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4473 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4474 skip "Need MDS version at least 2.7.56"
4476 ([ $FAILURE_MODE == "HARD" ] &&
4477 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4478 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4481 test_112_rename_prepare
4486 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4487 error "rename dir cross MDT failed!"
4492 rm -rf $DIR/$tdir || error "rmdir failed"
4494 run_test 112l "DNE: cross MDT rename, fail MDT1,MDT2,MDT4"
4497 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4498 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4499 skip "Need MDS version at least 2.7.56"
4501 ([ $FAILURE_MODE == "HARD" ] &&
4502 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4503 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4506 test_112_rename_prepare
4511 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4512 error "rename dir cross MDT failed!"
4517 rm -rf $DIR/$tdir || error "rmdir failed"
4519 run_test 112m "DNE: cross MDT rename, fail MDT1,MDT3,MDT4"
4522 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4523 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4524 skip "Need MDS version at least 2.7.56"
4526 ([ $FAILURE_MODE == "HARD" ] &&
4527 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4528 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4531 test_112_rename_prepare
4536 mrename $DIR/$tdir/src_dir/src_child $DIR/$tdir/tgt_dir/tgt_child ||
4537 error "rename dir cross MDT failed!"
4542 rm -rf $DIR/$tdir || error "rmdir failed"
4544 run_test 112n "DNE: cross MDT rename, fail MDT2,MDT3,MDT4"
4547 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4548 [[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
4549 skip "Need MDS version at least 2.7.56"
4551 ([ $FAILURE_MODE == "HARD" ] &&
4552 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4553 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4561 for ((j=0;j<$((MDSCOUNT));j++)); do
4562 fail_index=$((fail_index+1))
4563 index=$((fail_index % MDSCOUNT))
4564 replay_barrier mds$((index + 1))
4565 for ((i=0;i<5;i++)); do
4566 test_mkdir -i$index -c$MDSCOUNT $DIR/$tdir/test_$i ||
4567 error "create striped dir $DIR/$tdir/test_$i"
4570 fail mds$((index + 1))
4571 for ((i=0;i<5;i++)); do
4572 checkstat -t dir $DIR/$tdir/test_$i ||
4573 error "$DIR/$tdir/test_$i does not exist!"
4575 rm -rf $DIR/$tdir/test_* ||
4579 run_test 115 "failover for create/unlink striped directory"
4582 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4583 [ "$MDS1_VERSION" -lt $(version_code 2.7.55) ] &&
4584 skip "Do not support large update log before 2.7.55" &&
4586 ([ $FAILURE_MODE == "HARD" ] &&
4587 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4588 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4592 mkdir_on_mdt0 $DIR/$tdir
4595 # OBD_FAIL_SPLIT_UPDATE_REC 0x1702
4596 do_facet mds1 "lctl set_param fail_loc=0x80001702"
4597 $LFS setdirstripe -i0 -c$MDSCOUNT $DIR/$tdir/striped_dir
4600 $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
4601 error "stried_dir does not exists"
4603 run_test 116a "large update log master MDT recovery"
4606 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4607 [ "$MDS1_VERSION" -lt $(version_code 2.7.55) ] &&
4608 skip "Do not support large update log before 2.7.55" &&
4611 ([ $FAILURE_MODE == "HARD" ] &&
4612 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4613 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4617 mkdir_on_mdt0 $DIR/$tdir
4620 # OBD_FAIL_SPLIT_UPDATE_REC 0x1702
4621 do_facet mds2 "lctl set_param fail_loc=0x80001702"
4622 $LFS setdirstripe -i0 -c$MDSCOUNT $DIR/$tdir/striped_dir
4625 $CHECKSTAT -t dir $DIR/$tdir/striped_dir ||
4626 error "stried_dir does not exists"
4628 run_test 116b "large update log slave MDT recovery"
4631 [ $MDSCOUNT -lt 4 ] && skip "needs >= 4 MDTs" && return 0
4632 ([ $FAILURE_MODE == "HARD" ] &&
4633 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4634 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4640 $LFS setdirstripe -i0 -c$MDSCOUNT $DIR/$tdir/remote_dir
4641 $LFS setdirstripe -i1 -c$MDSCOUNT $DIR/$tdir/remote_dir_1
4644 # Let's set rdonly on all MDTs, so client will send
4645 # replay requests on all MDTs and replay these requests
4646 # at the same time. This test will verify the recovery
4647 # will not be deadlock in this case, LU-7531.
4648 for ((index = 0; index < $((MDSCOUNT)); index++)); do
4649 replay_barrier mds$((index + 1))
4650 if [ -z $mds_indexs ]; then
4651 mds_indexs="${mds_indexs}mds$((index+1))"
4653 mds_indexs="${mds_indexs},mds$((index+1))"
4657 rm -rf $DIR/$tdir/remote_dir
4658 rm -rf $DIR/$tdir/remote_dir_1
4662 rm -rf $DIR/$tdir || error "rmdir failed"
4664 run_test 117 "DNE: cross MDT unlink, fail MDT1 and MDT2"
4667 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4668 [ "$MDS1_VERSION" -lt $(version_code 2.7.64) ] &&
4669 skip "Do not support large update log before 2.7.64" &&
4674 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir ||
4675 error "setdirstripe fails"
4676 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1 ||
4677 error "setdirstripe fails 1"
4678 rm -rf $DIR/$tdir/striped_dir* || error "rmdir fails"
4680 # OBD_FAIL_INVALIDATE_UPDATE 0x1705
4681 do_facet mds1 "lctl set_param fail_loc=0x1705"
4682 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
4683 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
4684 do_facet mds1 "lctl set_param fail_loc=0x0"
4687 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir
4688 $LFS setdirstripe -c2 $DIR/$tdir/striped_dir1
4693 run_test 118 "invalidate osp update will not cause update log corruption"
4696 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4697 [ "$MDS1_VERSION" -lt $(version_code 2.7.64) ] &&
4698 skip "Do not support large update log before 2.7.64" &&
4701 local hard_timeout=$(do_facet mds1 \
4702 "lctl get_param -n mdt.$FSNAME-MDT0000.recovery_time_hard")
4704 local clients=${CLIENTS:-$HOSTNAME}
4705 local time_min=$(recovery_time_min)
4707 mkdir_on_mdt0 $DIR/$tdir
4708 mkdir $DIR/$tdir/tmp
4709 rmdir $DIR/$tdir/tmp
4712 mkdir $DIR/$tdir/dir_1
4713 for ((i = 0; i < 20; i++)); do
4714 $LFS setdirstripe -i0 -c2 $DIR/$tdir/stripe_dir-$i
4721 #define OBD_FAIL_TGT_REPLAY_DELAY 0x714
4722 do_facet mds1 $LCTL set_param fail_loc=0x80000714
4723 #sleep (timeout + 5), so mds will evict the client exports,
4724 #but DNE update recovery will keep going.
4725 do_facet mds1 $LCTL set_param fail_val=$((time_min + 5))
4727 mount_facet mds1 "-o recovery_time_hard=$time_min"
4729 wait_clients_import_state "$clients" mds1 FULL
4731 clients_up || clients_up || error "failover df: $?"
4733 #revert back the hard timeout
4734 do_facet mds1 $LCTL set_param \
4735 mdt.$FSNAME-MDT0000.recovery_time_hard=$hard_timeout
4737 for ((i = 0; i < 20; i++)); do
4738 stripe_count=$($LFS getdirstripe -c $DIR/$tdir/stripe_dir-$i)
4739 [ $stripe_count == 2 ] || {
4740 error "stripe_dir-$i creation replay fails"
4745 run_test 119 "timeout of normal replay does not cause DNE replay fails "
4748 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4749 [ "$MDS1_VERSION" -lt $(version_code 2.7.64) ] &&
4750 skip "Do not support large update log before 2.7.64" &&
4753 mkdir_on_mdt0 $DIR/$tdir
4754 replay_barrier_nosync mds1
4755 for ((i = 0; i < 20; i++)); do
4756 mkdir $DIR/$tdir/dir-$i || {
4757 error "create dir-$i fails"
4760 $LFS setdirstripe -i0 -c2 $DIR/$tdir/stripe_dir-$i || {
4761 error "create stripe_dir-$i fails"
4768 for ((i = 0; i < 20; i++)); do
4769 [ ! -e "$DIR/$tdir/dir-$i" ] || {
4770 error "dir-$i still exists"
4773 [ ! -e "$DIR/$tdir/stripe_dir-$i" ] || {
4774 error "stripe_dir-$i still exists"
4779 run_test 120 "DNE fail abort should stop both normal and DNE replay"
4782 [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] &&
4783 skip "Don't support it before 2.11" &&
4786 local at_max_saved=$(at_max_get mds)
4788 touch $DIR/$tfile || error "touch $DIR/$tfile failed"
4789 cancel_lru_locks mdc
4791 multiop_bg_pause $DIR/$tfile s_s || error "multiop $DIR/$tfile failed"
4794 lctl set_param -n ldlm.cancel_unused_locks_before_replay "0"
4800 #define OBD_FAIL_TGT_RECOVERY_REQ_RACE 0x721
4801 do_facet $SINGLEMDS "lctl set_param fail_loc=0x721 fail_val=0"
4805 wait_clients_import_state "$clients" mds1 FULL
4806 clients_up || clients_up || error "failover df: $?"
4809 wait $mpid || error "multiop_bg_pause pid failed"
4811 do_facet $SINGLEMDS "lctl set_param fail_loc=0x0"
4812 lctl set_param -n ldlm.cancel_unused_locks_before_replay "1"
4813 at_max_set $at_max_saved mds
4816 run_test 121 "lock replay timed out and race"
4819 [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] &&
4820 skip "Do not support Data-on-MDT before 2.11"
4822 replay_barrier $SINGLEMDS
4823 $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tfile
4826 [ $($LFS getstripe -L $DIR/$tfile) == "mdt" ] ||
4827 error "Fail to replay DoM file creation"
4829 run_test 130a "DoM file create (setstripe) replay"
4832 [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] &&
4833 skip "Do not support Data-on-MDT before 2.11"
4835 mkdir_on_mdt0 $DIR/$tdir
4836 $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tdir
4837 replay_barrier $SINGLEMDS
4838 touch $DIR/$tdir/$tfile
4841 [ $($LFS getstripe -L $DIR/$tdir/$tfile) == "mdt" ] ||
4842 error "Fail to replay DoM file creation"
4844 run_test 130b "DoM file create (inherited) replay"
4847 [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] &&
4848 skip "Do not support Data-on-MDT before 2.11"
4850 $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tfile
4851 replay_barrier $SINGLEMDS
4852 echo "dom_data" | dd of=$DIR/$tfile bs=8 count=1
4853 # lock is not canceled and will be replayed
4856 [ $(cat $DIR/$tfile) == "dom_data" ] ||
4857 error "Wrong file content after failover"
4859 run_test 131a "DoM file write lock replay"
4862 [ "$MDS1_VERSION" -lt $(version_code 2.10.90) ] &&
4863 skip "Do not support Data-on-MDT before 2.11"
4865 $LFS setstripe -E 1M -L mdt -E EOF -c 2 $DIR/$tfile
4866 replay_barrier $SINGLEMDS
4867 echo "dom_data" | dd of=$DIR/$tfile bs=8 count=1
4868 cancel_lru_locks mdc
4872 [ $(cat $DIR/$tfile) == "dom_data" ] ||
4873 error "Wrong file content after failover"
4875 run_test 131b "DoM file write replay"
4878 [ "$MDS1_VERSION" -lt $(version_code 2.12.0) ] &&
4879 skip "Need MDS version 2.12.0 or later"
4881 $LFS setstripe -E 1M -c 1 -E EOF -c 2 $DIR/$tfile
4882 replay_barrier $SINGLEMDS
4883 # write over the first component size cause next component instantiation
4884 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=1 seek=1 ||
4885 error "dd to $DIR/$tfile failed"
4886 lfs getstripe $DIR/$tfile
4888 cksum=$(md5sum $DIR/$tfile | awk '{print $1}')
4889 $LFS getstripe -I2 $DIR/$tfile | grep -q lmm_objects ||
4890 error "Component #1 was not instantiated"
4894 lfs getstripe $DIR/$tfile
4895 $LFS getstripe -I2 $DIR/$tfile | grep -q lmm_objects ||
4896 error "Component #1 instantiation was not replayed"
4897 cksum2=$(md5sum $DIR/$tfile | awk '{print $1}')
4898 if [ $cksum != $cksum2 ] ; then
4899 error_noexit "New cksum $cksum2 does not match original $cksum"
4902 run_test 132a "PFL new component instantiate replay"
4905 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
4906 ([ $FAILURE_MODE == "HARD" ] &&
4907 [ "$(facet_host mds1)" == "$(facet_host mds2)" ]) &&
4908 skip "MDTs needs to be on diff hosts for HARD fail mode" &&
4911 local remote_dir=$DIR/$tdir/remote_dir
4913 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
4914 $LFS mkdir -i 1 $remote_dir
4917 do_facet mds2 $LCTL set_param seq.srv*MDT0001.space=clear
4919 zconf_mount $(hostname) $MOUNT
4920 client_up || return 1
4922 #define OBD_FAIL_MDS_ALL_REQUEST_NET 0x123
4924 do_facet mds1 $LCTL set_param fail_val=700 fail_loc=0x80000123
4925 cp /etc/hosts $remote_dir/file &
4931 wait $pid || error "cp failed"
4932 rm -rf $DIR/$tdir || error "rmdir failed"
4936 run_test 133 "check resend of ongoing requests for lwp during failover"
4939 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return 0
4940 (( $MDS1_VERSION >= $(version_code 2.13.56) )) ||
4941 skip "need MDS version >= 2.13.56"
4944 pool_add_targets pool_134 1 1
4946 mkdir -p $DIR/$tdir/{A,B}
4947 $LFS setstripe -p pool_134 $DIR/$tdir/A
4948 $LFS setstripe -E EOF -p pool_134 $DIR/$tdir/B
4952 touch $DIR/$tdir/A/$tfile || error "touch non-pfl file failed"
4953 touch $DIR/$tdir/B/$tfile || error "touch pfl failed"
4957 [ -f $DIR/$tdir/A/$tfile ] || error "non-pfl file does not exist"
4958 [ -f $DIR/$tdir/B/$tfile ] || error "pfl file does not exist"
4960 run_test 134 "replay creation of a file created in a pool"
4964 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
4967 $LFS setstripe -S $((128 * 1024)) -i 0 $DIR/$tdir
4971 # Create 20 files so we have 20 ost locks
4972 for i in $(seq 20) ; do
4973 echo blah > $DIR/$tdir/file.${i}
4981 #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x32d
4982 # Make sure lock replay server side never completes and errors out.
4983 do_facet ost1 "$LCTL set_param fail_val=20"
4984 do_facet ost1 "$LCTL set_param fail_loc=0x32d"
4988 # Now make sure we notice
4991 sleep 20 # should we do something proactive to make reconnects go?
4992 kill -0 $PID || error "Unexpected sync success"
4999 do_facet ost1 "$LCTL set_param fail_loc=0"
5001 echo blah > $DIR/$tdir/file.test2
5005 run_test 135 "Server failure in lock replay phase"
5008 check_and_cleanup_lustre