5 LUSTRE=${LUSTRE:-`dirname $0`/..}
6 . $LUSTRE/tests/test-framework.sh
10 . ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
12 SETUP=${SETUP:-"setup"}
13 CLEANUP=${CLEANUP:-"cleanup"}
17 if [ "$MDSCOUNT" -gt 1 ]; then
19 for mds in `mds_list`; do
20 MDSDEV=$TMP/${mds}-`hostname`
21 add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1_svc
23 add_lov_to_lmv lov1 lmv1_svc --stripe_sz $STRIPE_BYTES \
24 --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
27 add_mds mds1 --dev $MDSDEV --size $MDSSIZE
28 add_lov lov1 mds1 --stripe_sz $STRIPE_BYTES \
29 --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
33 add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE --failover
34 add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE --failover
35 add_client client ${MDS} --lov lov1 --path $MOUNT
40 SETUP=${SETUP:-"setup"}
41 CLEANUP=${CLEANUP:-"cleanup"}
44 # make sure we are using the primary MDS, so the config log will
45 # be able to clean up properly.
46 activemds=`facet_active mds1`
47 if [ $activemds != "mds1" ]; then
51 umount $MOUNT2 || true
54 for mds in `mds_list`; do
55 stop $mds ${FORCE} $MDSLCONFARGS
60 stop ost ${FORCE} --dump cleanup-dual.log
63 if [ "$ONLY" == "cleanup" ]; then
64 sysctl -w portals.debug=0
72 start_krb5_kdc || exit 1
73 start ost --reformat $OSTLCONFARGS
74 PINGER=`cat /proc/fs/lustre/pinger`
76 if [ "$PINGER" != "on" ]; then
77 echo "ERROR: Lustre must be built with --enable-pinger for replay-dual"
82 start ost2 --reformat $OSTLCONFARGS
83 start_lsvcgssd || exit 2
85 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
86 for mds in `mds_list`; do
87 start $mds --reformat $MDSLCONFARGS
89 grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT
90 grep " $MOUNT2 " /proc/mounts || zconf_mount `hostname` $MOUNT2
92 echo $TIMEOUT > /proc/sys/lustre/timeout
93 echo $UPCALL > /proc/sys/lustre/upcall
97 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
106 checkstat $MOUNT2/a || return 1
107 checkstat $MOUNT1/b || return 2
108 rm $MOUNT2/a $MOUNT1/b
109 checkstat $MOUNT1/a && return 3
110 checkstat $MOUNT2/b && return 4
114 run_test 1 "|X| simple create"
122 checkstat $MOUNT2/adir || return 1
124 checkstat $MOUNT2/adir && return 2
128 run_test 2 "|X| mkdir adir"
133 mkdir $MOUNT2/adir/bdir
136 checkstat $MOUNT2/adir || return 1
137 checkstat $MOUNT1/adir/bdir || return 2
138 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
139 checkstat $MOUNT1/adir && return 3
140 checkstat $MOUNT2/adir/bdir && return 4
144 run_test 3 "|X| mkdir adir, mkdir adir/bdir "
149 mkdir $MOUNT1/adir && return 1
150 mkdir $MOUNT2/adir/bdir
153 checkstat $MOUNT2/adir || return 2
154 checkstat $MOUNT1/adir/bdir || return 3
156 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
157 checkstat $MOUNT1/adir && return 4
158 checkstat $MOUNT2/adir/bdir && return 5
162 run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir "
166 # multiclient version of replay_single.sh/test_8
168 multiop $MOUNT2/a o_tSc &
170 # give multiop a chance to open
175 wait $pid || return 1
178 [ -e $MOUNT2/a ] && return 2
181 run_test 5 "open, unlink |X| close"
186 multiop $MOUNT2/a o_c &
188 multiop $MOUNT1/a o_c &
190 # give multiop a chance to open
195 wait $pid1 || return 1
199 wait $pid2 || return 1
200 [ -e $MOUNT2/a ] && return 2
203 run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2"
207 multiop $MOUNT2/a o_c &
209 multiop $MOUNT1/a o_c &
211 # give multiop a chance to open
216 wait $pid2 || return 1
220 wait $pid1 || return 1
221 [ -e $MOUNT2/a ] && return 2
224 run_test 6b "open1, open2, unlink |X| close2 [fail mds] close1"
228 drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1
230 checkstat $MOUNT2/$tfile || return 2
231 rm $MOUNT1/$tfile || return 3
235 run_test 8 "replay of resent request"
239 mcreate $MOUNT1/$tfile-1
240 mcreate $MOUNT2/$tfile-2
241 # drop first reint reply
242 sysctl -w lustre.fail_loc=0x80000119
244 sysctl -w lustre.fail_loc=0
246 rm $MOUNT1/$tfile-[1,2] || return 1
250 run_test 9 "resending a replayed create"
253 mcreate $MOUNT1/$tfile-1
255 munlink $MOUNT1/$tfile-1
256 mcreate $MOUNT2/$tfile-2
257 # drop first reint reply
258 sysctl -w lustre.fail_loc=0x80000119
260 sysctl -w lustre.fail_loc=0
262 checkstat $MOUNT1/$tfile-1 && return 1
263 checkstat $MOUNT1/$tfile-2 || return 2
268 run_test 10 "resending a replayed unlink"
272 mcreate $MOUNT1/$tfile-1
273 mcreate $MOUNT2/$tfile-2
274 mcreate $MOUNT1/$tfile-3
275 mcreate $MOUNT2/$tfile-4
276 mcreate $MOUNT1/$tfile-5
277 # drop all reint replies for a while
278 sysctl -w lustre.fail_loc=0x0119
280 #sleep for while, let both clients reconnect and timeout
281 sleep $((TIMEOUT * 2))
282 sysctl -w lustre.fail_loc=0
284 rm $MOUNT1/$tfile-[1-5] || return 1
288 run_test 11 "both clients timeout during replay"
293 multiop $DIR/$tfile mo_c &
298 sysctl -w lustre.fail_loc=0x80000302
300 df $MOUNT || return 1
301 sysctl -w lustre.fail_loc=0
304 $CHECKSTAT -t file $DIR/$tfile || return 2
305 kill -USR1 $MULTIPID || return 3
306 wait $MULTIPID || return 4
311 run_test 12 "open resend timeout"
314 multiop $DIR/$tfile mo_c &
320 kill -USR1 $MULTIPID || return 3
321 wait $MULTIPID || return 4
324 sysctl -w lustre.fail_loc=0x80000115
326 df $MOUNT || return 1
327 sysctl -w lustre.fail_loc=0
330 $CHECKSTAT -t file $DIR/$tfile || return 2
335 run_test 13 "close resend timeout"
340 createmany -o $MOUNT1/$tfile- 25
341 createmany -o $MOUNT2/$tfile-2- 1
342 createmany -o $MOUNT1/$tfile-3- 25
346 # expect failover to fail
347 df $MOUNT && return 1
349 # first 25 files shouuld have been
352 unlinkmany $MOUNT1/$tfile- 25 || return 2
354 zconf_mount `hostname` $MOUNT2
357 run_test 14 "timeouts waiting for lost client during replay"
361 createmany -o $MOUNT1/$tfile- 25
362 createmany -o $MOUNT2/$tfile-2- 1
366 df $MOUNT || return 1
368 unlinkmany $MOUNT1/$tfile- 25 || return 2
370 zconf_mount `hostname` $MOUNT2
373 run_test 15 "timeout waiting for lost client during replay, 1 client completes"
376 createmany -o $MOUNT1/$tfile- 25
377 createmany -o $MOUNT2/$tfile-2- 1
383 df $MOUNT || return 1
385 unlinkmany $MOUNT1/$tfile- 25 || return 2
387 zconf_mount `hostname` $MOUNT2
391 #run_test 16 "fail MDS during recovery (3571)"
394 createmany -o $MOUNT1/$tfile- 25
395 createmany -o $MOUNT2/$tfile-2- 1
397 # Make sure the disconnect is lost
401 echo -1 > /proc/sys/portals/debug
405 df $MOUNT || return 1
407 unlinkmany $MOUNT1/$tfile- 25 || return 2
409 zconf_mount `hostname` $MOUNT2
413 #Still not support ost fail over
414 #run_test 17 "fail OST during recovery (3571)"
418 multiop $MOUNT2/$tfile O_c &
420 multiop $MOUNT1/$tfile O_c &
422 # give multiop a chance to open
430 zconf_mount `hostname` $MOUNT2
432 run_test 18 "replay open, Abort recovery, don't assert (3892)"
435 # cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for
438 test_20() { # bug 3822 - evicting client with enqueued lock
439 mkdir -p $MOUNT1/$tdir
440 touch $MOUNT1/$tdir/f0
441 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED 0x30b
442 statmany -s $MOUNT1/$tdir/f 500 &
445 do_facet mds1 sysctl -w lustre.fail_loc=0x8000030b # hold enqueue
447 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305
448 do_facet client sysctl -w lustre.fail_loc=0x80000305 # drop cb, evict
450 usleep 500 # wait to ensure first client is one that will be evicted
451 openfile -f O_RDONLY $MOUNT2/$tdir/f0
453 dmesg | grep "entering recovery in server" && \
454 error "client not evicted" || true
456 run_test 20 "ldlm_handle_enqueue succeeds on evicted export (3822)"
458 if [ "$ONLY" != "setup" ]; then
459 equals_msg test complete, cleaning up
461 SLEEP=$((`date +%s` - $NOW))
462 [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP