6 ALWAYS_EXCEPT="3c 4b 4c 10 $REPLAY_VBR_EXCEPT"
9 PTLDEBUG=${PTLDEBUG:--1}
10 LUSTRE=${LUSTRE:-`dirname $0`/..}
12 CLEANUP=${CLEANUP:-""}
13 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
19 [ -n "$CLIENTS" ] || { skip "Need two or more clients" && exit 0; }
20 [ $CLIENTCOUNT -ge 2 ] || \
21 { skip "Need two or more clients, have $CLIENTCOUNT" && exit 0; }
22 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
24 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
27 [ ! "$NAME" = "ncli" ] && ALWAYS_EXCEPT="$ALWAYS_EXCEPT"
28 [ "$NAME" = "ncli" ] && MOUNT_2=""
32 check_and_setup_lustre
33 rm -rf $DIR/[df][0-9]*
35 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
37 [ "$CLIENTS" ] && zconf_umount_clients $CLIENTS $DIR
40 echo "mount client $CLIENT1,$CLIENT2..."
41 zconf_mount_clients $CLIENT1 $DIR
42 zconf_mount_clients $CLIENT2 $DIR
44 do_node $CLIENT2 mkdir -p $DIR/$tdir
46 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
47 do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 1
48 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
49 zconf_umount $CLIENT2 $DIR
52 # recovery shouldn't fail due to missing client 2
53 do_node $CLIENT1 df $DIR || return 1
55 # All 50 files should have been replayed
56 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
57 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
59 zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
60 [ -e $DIR/$tdir/$tfile-2-0 ] && error "$tfile-2-0 exists"
62 zconf_umount_clients $CLIENTS $DIR
65 run_test 1 "VBR: client during replay doesn't affect another one"
68 #ls -al $DIR/$tdir/$tfile
70 zconf_mount_clients $CLIENT1 $DIR
71 zconf_mount_clients $CLIENT2 $DIR
73 do_node $CLIENT2 mkdir -p $DIR/$tdir
75 do_node $CLIENT2 mcreate $DIR/$tdir/$tfile
76 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
77 #do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 1
78 do_node $CLIENT1 $CHECKSTAT $DIR/$tdir/$tfile
79 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
80 zconf_umount $CLIENT2 $DIR
83 # recovery shouldn't fail due to missing client 2
84 do_node $CLIENT1 df $DIR || return 1
86 # All 50 files should have been replayed
87 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
88 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
90 do_node $CLIENT1 $CHECKSTAT $DIR/$tdir/$tfile && return 4
92 zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
94 zconf_umount_clients $CLIENTS $DIR
97 run_test 2 "VBR: lost data due to missed REMOTE client during replay"
100 zconf_mount_clients $CLIENT1 $DIR
101 zconf_mount_clients $CLIENT2 $DIR
103 #make sure the time will change
104 do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0" || return
105 do_node $CLIENT1 touch $DIR/$tfile
106 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
110 do_node $CLIENT2 touch $DIR/$tfile
111 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
113 do_node $CLIENT1 touch $DIR/$tfile
115 do_node $CLIENT2 rm $DIR/$tfile
116 zconf_umount $CLIENT2 $DIR
119 # recovery shouldn't fail due to missing client 2
120 do_node $CLIENT1 df $DIR || return 1
121 do_node $CLIENT1 $CHECKSTAT $DIR/$tfile && return 2
123 zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
125 zconf_umount_clients $CLIENTS $DIR
129 run_test 3a "VBR: setattr of time/size doesn't change version"
132 zconf_mount_clients $CLIENT1 $DIR
133 zconf_mount_clients $CLIENT2 $DIR
135 #make sure the time will change
136 do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0" || return
137 do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0" || return
138 do_node $CLIENT1 touch $DIR/$tfile
139 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
143 do_node $CLIENT2 chmod +x $DIR/$tfile
144 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
146 do_node $CLIENT1 chmod -x $DIR/$tfile
147 zconf_umount $CLIENT2 $DIR
150 # recovery should fail due to missing client 2
151 do_node $CLIENT1 df $DIR && return 1
153 do_node $CLIENT1 $CHECKSTAT -p 755 $DIR/$tfile && return 2
154 zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
156 zconf_umount_clients $CLIENTS $DIR
160 run_test 3b "VBR: setattr of permissions changes version"
163 [ "$FAILURE_MODE" = HARD ] || \
164 { skip "The HARD failure is needed" && return 0; }
166 [ $RUNAS_ID -eq $UID ] && skip "RUNAS_ID = UID = $UID -- skipping" && return
168 zconf_mount_clients $CLIENT1 $DIR
169 zconf_mount_clients $CLIENT2 $DIR
171 # check that permission changes are synced
172 do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=1"
174 do_node $CLIENT1 mkdir -p $DIR/d3c/sub || error
175 #chown -R $RUNAS_ID $MOUNT1/d3
176 do_node $CLIENT1 ls -la $DIR/d3c
178 # only HARD failure will work as we use sync operation
180 do_node $CLIENT2 mcreate $DIR/d3c/$tfile-2
182 do_node $CLIENT1 chmod 0700 $UID $DIR/d3c
184 do_node $CLIENT1 mcreate $DIR/d3c/sub/$tfile
185 do_node $CLIENT1 echo "Top Secret" > $DIR/d3c/sub/$tfile
186 #check user can't access new file
187 do_node $CLIENT2 $RUNAS ls $DIR/d3c && return 3
188 do_node $CLIENT1 $RUNAS ls $DIR/d3c && return 4
189 do_node $CLIENT1 $RUNAS cat $DIR/d3c/sub/$tfile && return 5
191 zconf_umount $CLIENT2 $DIR
194 # recovery shouldn't fail due to missing client 2
195 do_node $CLIENT1 df $DIR || return 1
198 zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
199 do_node $CLIENT1 $RUNAS cat $DIR/d3c/sub/$tfile && return 6
200 do_node $CLIENT2 $RUNAS cat $DIR/d3c/sub/$tfile && return 7
201 do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
205 run_test 3c "VBR: permission dependency failure"
207 vbr_deactivate_client() {
209 echo "Deactivating client $client";
210 do_node $client "sysctl -w lustre.fail_loc=0x50d"
213 vbr_activate_client() {
215 echo "Activating client $client";
216 do_node $client "sysctl -w lustre.fail_loc=0x0"
222 [ -z "$(do_node $client lctl dl | grep mdt)" ] && \
223 [ -z "$(do_node $client lctl dl | grep ost)" ]
227 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
229 remote_server $CLIENT2 || \
230 { skip "Client $CLIENT2 is on the server node" && return 0; }
232 zconf_mount_clients $CLIENT1 $DIR
233 zconf_mount_clients $CLIENT2 $DIR
235 do_node $CLIENT2 mkdir -p $DIR/$tdir
237 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
238 do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 25
239 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
240 vbr_deactivate_client $CLIENT2
243 do_node $CLIENT1 df $DIR || return 1
245 # All 50 files should have been replayed
246 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
247 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
249 vbr_activate_client $CLIENT2
250 do_node $CLIENT2 df $DIR || return 4
251 # All 25 files from client2 should have been replayed
252 do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
254 zconf_umount_clients $CLIENTS $DIR
257 run_test 4a "fail MDS, delayed recovery"
260 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
262 remote_server $CLIENT2 || \
263 { skip "Client $CLIENT2 is on the server node" && return 0; }
265 zconf_mount_clients $CLIENT1 $DIR
266 zconf_mount_clients $CLIENT2 $DIR
269 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
270 do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 25
271 vbr_deactivate_client $CLIENT2
274 do_node $CLIENT1 df $DIR || return 1
276 # create another set of files
277 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
279 vbr_activate_client $CLIENT2
280 do_node $CLIENT2 df $DIR || return 2
282 # All files from should have been replayed
283 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
284 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 4
285 do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
287 zconf_umount_clients $CLIENTS $DIR
289 run_test 4b "fail MDS, normal operation, delayed open recovery"
292 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
294 remote_server $CLIENT2 || \
295 { skip "Client $CLIENT2 is on the server node" && return 0; }
297 zconf_mount_clients $CLIENT1 $DIR
298 zconf_mount_clients $CLIENT2 $DIR
301 do_node $CLIENT1 createmany -m $DIR/$tfile- 25
302 do_node $CLIENT2 createmany -m $DIR/$tdir/$tfile-2- 25
303 vbr_deactivate_client $CLIENT2
306 do_node $CLIENT1 df $DIR || return 1
308 # create another set of files
309 do_node $CLIENT1 createmany -m $DIR/$tfile-3- 25
311 vbr_activate_client $CLIENT2
312 do_node $CLIENT2 df $DIR || return 2
314 # All files from should have been replayed
315 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
316 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 4
317 do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
319 zconf_umount_clients $CLIENTS $DIR
321 run_test 4c "fail MDS, normal operation, delayed recovery"
324 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
326 remote_server $CLIENT2 || \
327 { skip "Client $CLIENT2 is on the server node" && return 0; }
329 zconf_mount_clients $CLIENT1 $DIR
330 zconf_mount_clients $CLIENT2 $DIR
333 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
334 do_node $CLIENT2 createmany -o $DIR/$tfile-2- 1
335 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 1
336 vbr_deactivate_client $CLIENT2
339 do_node $CLIENT1 df $DIR && return 1
341 vbr_activate_client $CLIENT2
342 do_node $CLIENT2 df $DIR || return 2
344 # First 25 files should have been replayed
345 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
346 # Third file is failed due to missed client2
347 do_node $CLIENT1 $CHECKSTAT $DIR/$tfile-3-0 && error "$tfile-3-0 exists"
348 # file from client2 should exists
349 do_node $CLIENT2 unlinkmany $DIR/$tfile-2- 1 || return 4
351 zconf_umount_clients $CLIENTS $DIR
353 run_test 5a "fail MDS, delayed recovery should fail"
356 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
358 remote_server $CLIENT2 || \
359 { skip "Client $CLIENT2 is on the server node" && return 0; }
361 zconf_mount_clients $CLIENT1 $DIR
362 zconf_mount_clients $CLIENT2 $DIR
365 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
366 do_node $CLIENT2 createmany -o $DIR/$tfile-2- 1
367 vbr_deactivate_client $CLIENT2
370 do_node $CLIENT1 df $DIR || return 1
371 do_node $CLIENT1 $CHECKSTAT $DIR/$tfile-2-0 && error "$tfile-2-0 exists"
373 # create another set of files
374 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
376 vbr_activate_client $CLIENT2
377 do_node $CLIENT2 df $DIR && return 4
378 # file from client2 should fail
379 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile-2-0 && error "$tfile-2-0 exists"
381 # All 50 files from client 1 should have been replayed
382 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
383 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
385 zconf_umount_clients $CLIENTS $DIR
387 run_test 5b "fail MDS, normal operation, delayed recovery should fail"
390 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
392 remote_server $CLIENT2 || \
393 { skip "Client $CLIENT2 is on the server node" && return 0; }
395 zconf_mount_clients $CLIENT1 $DIR
396 zconf_mount_clients $CLIENT2 $DIR
398 do_node $CLIENT2 mkdir -p $DIR/$tdir
400 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
401 do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 25
402 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
403 vbr_deactivate_client $CLIENT2
406 # replay only 5 requests
407 do_node $CLIENT2 "sysctl -w lustre.fail_val=5"
408 #define OBD_FAIL_PTLRPC_REPLAY 0x50e
409 do_node $CLIENT2 "sysctl -w lustre.fail_loc=0x2000050e"
410 do_node $CLIENT2 df $DIR
411 # vbr_activate_client $CLIENT2
412 # need way to know that client stops replays
416 do_node $CLIENT1 df $DIR || return 1
418 # All files should have been replayed
419 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
420 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
421 do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
423 zconf_umount_clients $CLIENTS $DIR
426 run_test 6a "fail MDS, delayed recovery, fail MDS"
429 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
431 remote_server $CLIENT2 || \
432 { skip "Client $CLIENT2 is on the server node" && return 0; }
434 zconf_mount_clients $CLIENT1 $DIR
435 zconf_mount_clients $CLIENT2 $DIR
437 do_node $CLIENT2 mkdir -p $DIR/$tdir
439 do_node $CLIENT1 createmany -o $DIR/$tfile- 25
440 do_node $CLIENT2 createmany -o $DIR/$tdir/$tfile-2- 25
441 do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
442 vbr_deactivate_client $CLIENT2
445 vbr_activate_client $CLIENT2
446 do_node $CLIENT2 df $DIR || return 4
449 do_node $CLIENT1 df $DIR || return 1
451 # All files should have been replayed
452 do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
453 do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
454 do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
456 zconf_umount_clients $CLIENTS $DIR
459 run_test 7a "fail MDS, delayed recovery, fail MDS"
465 # We need to run do_node in bg, because pdsh does not exit
466 # if child process of run script exists.
467 # I.e. pdsh does not exit when runmultiop_bg_pause exited,
468 # because of multiop_bg_pause -> $MULTIOP_PROG &
469 # By the same reason we need sleep a bit after do_nodes starts
470 # to let runmultiop_bg_pause start muliop and
471 # update /tmp/multiop_bg.pid ;
472 # The rm /tmp/multiop_bg.pid guarantees here that
473 # we have the updated by runmultiop_bg_pause
474 # /tmp/multiop_bg.pid file
476 local pid_file=$TMP/multiop_bg.pid.$$
477 do_node $client "rm -f $pid_file && MULTIOP_PID_FILE=$pid_file LUSTRE= runmultiop_bg_pause $file O_tSc" &
481 multiop_pid=$(do_node $client cat $pid_file)
482 [ -n "$multiop_pid" ] || error "$client : Can not get multiop_pid from $pid_file "
483 eval export ${client}_multiop_pid=$multiop_pid
484 eval export ${client}_do_node_pid=$pid
485 local var=${client}_multiop_pid
486 echo client $client multiop_bg started multiop_pid=${!var}
492 local multiop_pid=${client}_multiop_pid
493 local do_node_pid=${client}_do_node_pid
495 echo "Stopping multiop_pid=${!multiop_pid} (kill ${!multiop_pid} on $client)"
496 do_node $client kill -USR1 ${!multiop_pid}
498 wait ${!do_node_pid} || true
502 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
504 remote_server $CLIENT2 || \
505 { skip "Client $CLIENT2 is on the server node" && return 0; }
507 zconf_mount_clients $CLIENT1 $DIR
508 zconf_mount_clients $CLIENT2 $DIR
510 rmultiop_start $CLIENT2 $DIR/$tfile || return 1
511 do_node $CLIENT2 rm -f $DIR/$tfile
513 rmultiop_stop $CLIENT2 || return 2
515 vbr_deactivate_client $CLIENT2
517 do_node $CLIENT1 df $DIR || return 3
518 #client1 is back and will try to open orphan
519 vbr_activate_client $CLIENT2
520 do_node $CLIENT2 df $DIR || return 4
522 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
523 zconf_umount_clients $CLIENTS $DIR
526 run_test 8a "orphans are kept until delayed recovery"
529 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
531 remote_server $CLIENT2 || \
532 { skip "Client $CLIENT2 is on the server node" && return 0; }
534 zconf_mount_clients $CLIENT1 $DIR
535 zconf_mount_clients $CLIENT2 $DIR
537 rmultiop_start $CLIENT2 $DIR/$tfile || return 1
539 do_node $CLIENT1 rm -f $DIR/$tfile
541 vbr_deactivate_client $CLIENT2
543 do_node $CLIENT1 df $DIR || return 2
544 #client1 is back and will try to open orphan
545 vbr_activate_client $CLIENT2
546 do_node $CLIENT2 df $DIR || return 3
548 rmultiop_stop $CLIENT2 || return 1
549 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
550 zconf_umount_clients $CLIENTS $DIR
553 run_test 8b "open1 | unlink2 X delayed_replay1, close1"
556 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
558 remote_server $CLIENT2 || \
559 { skip "Client $CLIENT2 is on the server node" && return 0; }
561 zconf_mount_clients $CLIENT1 $DIR
562 zconf_mount_clients $CLIENT2 $DIR
564 rmultiop_start $CLIENT2 $DIR/$tfile || return 1
566 do_node $CLIENT1 rm -f $DIR/$tfile
567 rmultiop_stop $CLIENT2 || return 2
569 vbr_deactivate_client $CLIENT2
571 do_node $CLIENT1 df $DIR || return 3
572 #client1 is back and will try to open orphan
573 vbr_activate_client $CLIENT2
574 do_node $CLIENT2 df $DIR || return 4
576 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
577 zconf_umount_clients $CLIENTS $DIR
580 run_test 8c "open1 | unlink2, close1 X delayed_replay1"
583 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
585 remote_server $CLIENT2 || \
586 { skip "Client $CLIENT2 is on the server node" && return 0; }
588 zconf_mount_clients $CLIENT1 $DIR
589 zconf_mount_clients $CLIENT2 $DIR
591 rmultiop_start $CLIENT1 $DIR/$tfile || return 1
592 rmultiop_start $CLIENT2 $DIR/$tfile || return 2
594 do_node $CLIENT1 rm -f $DIR/$tfile
595 rmultiop_stop $CLIENT2 || return 3
596 rmultiop_stop $CLIENT1 || return 4
598 vbr_deactivate_client $CLIENT2
600 do_node $CLIENT1 df $DIR || return 6
602 #client1 is back and will try to open orphan
603 vbr_activate_client $CLIENT2
604 do_node $CLIENT2 df $DIR || return 8
606 do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
607 zconf_umount_clients $CLIENTS $DIR
610 run_test 8d "open1, open2 | unlink2, close1, close2 X delayed_replay1"
613 zconf_mount $CLIENT1 $DIR
614 zconf_mount $CLIENT2 $DIR
616 do_node $CLIENT1 mcreate $DIR/$tfile
617 do_node $CLIENT1 mkdir $DIR/$tfile-2
619 # missed replay from client1 will lead to recovery by versions
620 do_node $CLIENT1 touch $DIR/$tfile-2/$tfile
621 do_node $CLIENT2 rm $DIR/$tfile || return 1
622 do_node $CLIENT2 touch $DIR/$tfile || return 2
624 zconf_umount $CLIENT1 $DIR
626 do_node $CLIENT2 df $DIR || return 6
628 do_node $CLIENT2 rm $DIR/$tfile || error "$tfile doesn't exists"
629 zconf_umount_clients $CLIENTS $DIR
632 run_test 8e "create | unlink, create shouldn't fail"
635 zconf_mount_clients $CLIENT1 $DIR
636 zconf_mount_clients $CLIENT2 $DIR
638 do_node $CLIENT1 touch $DIR/$tfile
639 do_node $CLIENT1 mkdir $DIR/$tfile-2
641 # missed replay from client1 will lead to recovery by versions
642 do_node $CLIENT1 touch $DIR/$tfile-2/$tfile
643 do_node $CLIENT2 rm -f $DIR/$tfile || return 1
644 do_node $CLIENT2 mcreate $DIR/$tfile || return 2
646 zconf_umount $CLIENT1 $DIR
648 do_node $CLIENT2 df $DIR || return 6
650 do_node $CLIENT2 rm $DIR/$tfile || error "$tfile doesn't exists"
651 zconf_umount $CLIENT2 $DIR
654 run_test 8f "create | unlink, create shouldn't fail"
657 zconf_mount_clients $CLIENT1 $DIR
658 zconf_mount_clients $CLIENT2 $DIR
660 do_node $CLIENT1 touch $DIR/$tfile
661 do_node $CLIENT1 mkdir $DIR/$tfile-2
663 # missed replay from client1 will lead to recovery by versions
664 do_node $CLIENT1 touch $DIR/$tfile-2/$tfile
665 do_node $CLIENT2 rm -f $DIR/$tfile || return 1
666 do_node $CLIENT2 mkdir $DIR/$tfile || return 2
668 zconf_umount $CLIENT1 $DIR
670 do_node $CLIENT2 df $DIR || return 6
672 do_node $CLIENT2 rmdir $DIR/$tfile || error "$tfile doesn't exists"
673 zconf_umount $CLIENT2 $DIR
676 run_test 8g "create | unlink, create shouldn't fail"
679 delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
681 [ -z "$DBENCH_LIB" ] && skip "DBENCH_LIB is not set" && return 0
683 zconf_mount_clients $CLIENTS $DIR
685 local duration="-t 60"
686 local cmd="rundbench 1 $duration "
688 for CLIENT in ${CLIENTS//,/ }; do
689 $PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
691 echo $PID >pid.$CLIENT
692 echo "Started load PID=`cat pid.$CLIENT`"
696 sleep 3 # give clients a time to do operations
698 vbr_deactivate_client $CLIENT2
700 log "$TESTNAME fail mds 1"
703 # wait for client to reconnect to MDS
706 vbr_activate_client $CLIENT2
707 do_node $CLIENT2 df $DIR || return 4
709 for CLIENT in ${CLIENTS//,/ }; do
710 PID=`cat pid.$CLIENT`
713 echo "load on ${CLIENT} returned $rc"
716 zconf_umount_clients $CLIENTS $DIR
718 run_test 10 "mds version recovery; $CLIENTCOUNT clients"
720 equals_msg `basename $0`: test complete, cleaning up
721 #SLEEP=$((`date +%s` - $NOW))
722 #[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
723 check_and_cleanup_lustre
724 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true