5 LUSTRE=${LUSTRE:-`dirname $0`/..}
6 . $LUSTRE/tests/test-framework.sh
10 . ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
14 if [ "$MDSCOUNT" -gt 1 ]; then
16 for mds in `mds_list`; do
17 MDSDEV=$TMP/${mds}-`hostname`
18 add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1
20 add_lov_to_lmv lov1 lmv1 --stripe_sz $STRIPE_BYTES \
21 --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
24 add_mds mds1 --dev $MDSDEV --size $MDSSIZE
25 add_lov lov1 mds1 --stripe_sz $STRIPE_BYTES \
26 --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
31 add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
32 add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE
33 add_client client --mds ${MDS} --lov lov1 --path $MOUNT
40 SETUP=${SETUP:-"setup"}
41 CLEANUP=${CLEANUP:-"cleanup"}
44 # make sure we are using the primary MDS, so the config log will
45 # be able to clean up properly.
46 activemds=`facet_active mds1`
47 if [ $activemds != "mds1" ]; then
51 umount $MOUNT2 || true
54 for mds in `mds_list`; do
55 stop $mds ${FORCE} $MDSLCONFARGS
58 stop ost ${FORCE} --dump cleanup-dual.log
61 if [ "$ONLY" == "cleanup" ]; then
62 sysctl -w portals.debug=0
69 start ost --reformat $OSTLCONFARGS
70 PINGER=`cat /proc/fs/lustre/pinger`
72 if [ "$PINGER" != "on" ]; then
73 echo "ERROR: Lustre must be built with --enable-pinger for replay-dual"
78 start ost2 --reformat $OSTLCONFARGS
79 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
80 for mds in `mds_list`; do
81 start $mds --reformat $MDSLCONFARGS
83 grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT
84 grep " $MOUNT2 " /proc/mounts || zconf_mount `hostname` $MOUNT2
86 echo $TIMEOUT > /proc/sys/lustre/timeout
87 echo $UPCALL > /proc/sys/lustre/upcall
91 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
100 checkstat $MOUNT2/a || return 1
101 checkstat $MOUNT1/b || return 2
102 rm $MOUNT2/a $MOUNT1/b
103 checkstat $MOUNT1/a && return 3
104 checkstat $MOUNT2/b && return 4
108 run_test 1 "|X| simple create"
116 checkstat $MOUNT2/adir || return 1
118 checkstat $MOUNT2/adir && return 2
122 run_test 2 "|X| mkdir adir"
127 mkdir $MOUNT2/adir/bdir
130 checkstat $MOUNT2/adir || return 1
131 checkstat $MOUNT1/adir/bdir || return 2
132 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
133 checkstat $MOUNT1/adir && return 3
134 checkstat $MOUNT2/adir/bdir && return 4
138 run_test 3 "|X| mkdir adir, mkdir adir/bdir "
143 mkdir $MOUNT1/adir && return 1
144 mkdir $MOUNT2/adir/bdir
147 checkstat $MOUNT2/adir || return 2
148 checkstat $MOUNT1/adir/bdir || return 3
150 rmdir $MOUNT2/adir/bdir $MOUNT1/adir
151 checkstat $MOUNT1/adir && return 4
152 checkstat $MOUNT2/adir/bdir && return 5
156 run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir "
160 # multiclient version of replay_single.sh/test_8
162 multiop $MOUNT2/a o_tSc &
164 # give multiop a chance to open
169 wait $pid || return 1
172 [ -e $MOUNT2/a ] && return 2
175 run_test 5 "open, unlink |X| close"
180 multiop $MOUNT2/a o_c &
182 multiop $MOUNT1/a o_c &
184 # give multiop a chance to open
189 wait $pid1 || return 1
193 wait $pid2 || return 1
194 [ -e $MOUNT2/a ] && return 2
197 run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2"
201 multiop $MOUNT2/a o_c &
203 multiop $MOUNT1/a o_c &
205 # give multiop a chance to open
210 wait $pid2 || return 1
214 wait $pid1 || return 1
215 [ -e $MOUNT2/a ] && return 2
218 run_test 6b "open1, open2, unlink |X| close2 [fail mds] close1"
222 createmany -o $MOUNT1/$tfile- 25
223 createmany -o $MOUNT2/$tfile-2- 1
224 createmany -o $MOUNT1/$tfile-3- 25
228 # expect failover to fail
229 df $MOUNT && return 1
231 # 3313 - current fix for 3313 prevents any reply here
232 # unlinkmany $MOUNT1/$tfile- 25 || return 2
234 zconf_mount `hostname` $MOUNT2
237 run_test 7 "timeouts waiting for lost client during replay"
242 drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1
244 checkstat $MOUNT2/$tfile || return 2
245 rm $MOUNT1/$tfile || return 3
249 run_test 8 "replay of resent request"
253 mcreate $MOUNT1/$tfile-1
254 mcreate $MOUNT2/$tfile-2
255 # drop first reint reply
256 sysctl -w lustre.fail_loc=0x80000119
258 sysctl -w lustre.fail_loc=0
260 rm $MOUNT1/$tfile-[1,2] || return 1
264 run_test 9 "resending a replayed create"
267 mcreate $MOUNT1/$tfile-1
269 munlink $MOUNT1/$tfile-1
270 mcreate $MOUNT2/$tfile-2
271 # drop first reint reply
272 sysctl -w lustre.fail_loc=0x80000119
274 sysctl -w lustre.fail_loc=0
276 checkstat $MOUNT1/$tfile-1 && return 1
277 checkstat $MOUNT1/$tfile-2 || return 2
282 run_test 10 "resending a replayed unlink"
286 mcreate $MOUNT1/$tfile-1
287 mcreate $MOUNT2/$tfile-2
288 mcreate $MOUNT1/$tfile-3
289 mcreate $MOUNT2/$tfile-4
290 mcreate $MOUNT1/$tfile-5
291 # drop all reint replies for a while
292 sysctl -w lustre.fail_loc=0x0119
294 #sleep for while, let both clients reconnect and timeout
295 sleep $((TIMEOUT * 2))
296 sysctl -w lustre.fail_loc=0
298 rm $MOUNT1/$tfile-[1-5] || return 1
302 run_test 11 "both clients timeout during replay"
307 multiop $DIR/$tfile mo_c &
312 sysctl -w lustre.fail_loc=0x80000302
314 df $MOUNT || return 1
315 sysctl -w lustre.fail_loc=0
318 $CHECKSTAT -t file $DIR/$tfile || return 2
319 kill -USR1 $MULTIPID || return 3
320 wait $MULTIPID || return 4
325 run_test 12 "open resend timeout"
328 multiop $DIR/$tfile mo_c &
334 kill -USR1 $MULTIPID || return 3
335 wait $MULTIPID || return 4
338 sysctl -w lustre.fail_loc=0x80000115
340 df $MOUNT || return 1
341 sysctl -w lustre.fail_loc=0
344 $CHECKSTAT -t file $DIR/$tfile || return 2
349 run_test 13 "close resend timeout"
353 multiop $MOUNT2/$tfile O_c &
355 multiop $MOUNT1/$tfile O_c &
357 # give multiop a chance to open
365 zconf_mount `hostname` $MOUNT2
367 run_test 20 "replay open, Abort recovery, don't assert (3892)"
369 if [ "$ONLY" != "setup" ]; then
370 equals_msg test complete, cleaning up