2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 # Tests for multiple mount protection (MMP) feature.
6 # Run select tests by setting ONLY, or as arguments to the script.
7 # Skip specific tests by setting EXCEPT.
9 # e.g. ONLY="5 6" or ONLY="`seq 8 11`" or EXCEPT="7"
14 # bug number for skipped test:
15 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$MMP_EXCEPT"}
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 SRCDIR=$(cd $(dirname $0); echo $PWD)
19 export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH:/sbin
21 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
22 . $LUSTRE/tests/test-framework.sh
24 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
27 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
28 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
30 # unmount and cleanup the Lustre filesystem
31 MMP_RESTORE_MOUNT=false
32 if is_mounted $MOUNT || is_mounted $MOUNT2; then
34 MMP_RESTORE_MOUNT=true
37 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
42 # Get the failover facet.
43 get_failover_facet() {
45 local failover_facet=${facet}failover
47 local host=$(facet_host $facet)
48 local failover_host=$(facet_host $failover_facet)
50 [ -z "$failover_host" -o "$host" = "$failover_host" ] && \
56 # Initiate the variables for Lustre servers and targets.
58 MMP_MDS=${MMP_MDS:-$SINGLEMDS}
59 MMP_MDS_FAILOVER=$(get_failover_facet $MMP_MDS)
61 local mds_num=$(echo $MMP_MDS | tr -d "mds")
62 MMP_MDSDEV=$(mdsdevname $mds_num)
64 MMP_OSS=${MMP_OSS:-ost1}
65 MMP_OSS_FAILOVER=$(get_failover_facet $MMP_OSS)
67 local oss_num=$(echo $MMP_OSS | tr -d "ost")
68 MMP_OSTDEV=$(ostdevname $oss_num)
71 # Stop the MDS and OSS services on the primary or failover servers.
79 if [ "$flavor" = "failover" ]; then
80 mds_facet=$MMP_MDS_FAILOVER
81 oss_facet=$MMP_OSS_FAILOVER
87 stop $mds_facet $opts || return ${PIPESTATUS[0]}
88 stop $oss_facet $opts || return ${PIPESTATUS[0]}
91 # Enable the MMP feature.
96 do_facet $facet "$TUNE2FS -O mmp $device"
97 return ${PIPESTATUS[0]}
100 # Disable the MMP feature.
105 do_facet $facet "$TUNE2FS -O ^mmp $device"
106 return ${PIPESTATUS[0]}
109 # Reset the MMP block (if any) back to the clean state.
114 do_facet $facet "$TUNE2FS -f -E clear-mmp $device"
115 return ${PIPESTATUS[0]}
118 # Check whether the MMP feature is enabled or not.
123 do_facet $facet "$DUMPE2FS -h $device | grep mmp"
124 return ${PIPESTATUS[0]}
127 # Get MMP update interval (in seconds) from the Lustre server target.
128 get_mmp_update_interval() {
133 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
134 | grep 'MMP Update Interval' | cut -d' ' -f4")
135 [ -z "$interval" ] && interval=1
140 # Get MMP check interval (in seconds) from the Lustre server target.
141 get_mmp_check_interval() {
146 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
147 | grep 'MMP Check Interval' | cut -d' ' -f4")
148 [ -z "$interval" ] && interval=5
153 # Enable the MMP feature on the Lustre server targets.
157 # The MMP feature is automatically enabled by mkfs.lustre for
158 # new file system at format time if failover is being used.
159 # Otherwise, the Lustre administrator has to manually enable
160 # this feature when the file system is unmounted.
162 local var=${MMP_MDS}failover_HOST
163 if [ -z "${!var}" ]; then
164 log "Failover is not used on MDS, enabling MMP manually..."
165 enable_mmp $MMP_MDS $MMP_MDSDEV || \
166 error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS"
169 var=${MMP_OSS}failover_HOST
170 if [ -z "${!var}" ]; then
171 log "Failover is not used on OSS, enabling MMP manually..."
172 enable_mmp $MMP_OSS $MMP_OSTDEV || \
173 error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS"
176 # check whether the MMP feature is enabled or not
177 mmp_is_enabled $MMP_MDS $MMP_MDSDEV || \
178 error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS"
180 mmp_is_enabled $MMP_OSS $MMP_OSTDEV || \
181 error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS"
184 # Disable the MMP feature on the Lustre server targets
185 # which did not use failover.
188 if [ -z "$mdsfailover_HOST" ]; then
189 log "Failover is not used on MDS, disabling MMP manually..."
190 disable_mmp $MMP_MDS $MMP_MDSDEV || \
191 error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS"
192 mmp_is_enabled $MMP_MDS $MMP_MDSDEV && \
193 error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS"
196 if [ -z "$ostfailover_HOST" ]; then
197 log "Failover is not used on OSS, disabling MMP manually..."
198 disable_mmp $MMP_OSS $MMP_OSTDEV || \
199 error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS"
200 mmp_is_enabled $MMP_OSS $MMP_OSTDEV && \
201 error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS"
207 # Mount the shared target on the failover server after some interval it's
208 # mounted on the primary server.
209 mount_after_interval_sub() {
217 local failover_facet=$(get_failover_facet $facet)
220 local first_mount_rc=0
221 local second_mount_rc=0
223 log "Mounting $device on $facet..."
224 start $facet $device $opts &
227 if [ $interval -ne 0 ]; then
228 log "sleep $interval..."
232 log "Mounting $device on $failover_facet..."
233 start $failover_facet $device $opts
234 second_mount_rc=${PIPESTATUS[0]}
237 first_mount_rc=${PIPESTATUS[0]}
239 if [ $second_mount_rc -eq 0 -a $first_mount_rc -eq 0 ]; then
240 error_noexit "one mount delayed by mmp interval $interval should fail"
241 stop $facet || return ${PIPESTATUS[0]}
242 [ "$failover_facet" != "$facet" ] && stop $failover_facet || \
243 return ${PIPESTATUS[0]}
245 elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
246 error_noexit "failed to mount on the failover pair $facet,$failover_facet"
247 return $first_mount_rc
253 mount_after_interval() {
254 local mdt_interval=$1
255 local ost_interval=$2
258 mount_after_interval_sub $mdt_interval $MMP_MDSDEV $MMP_MDS \
259 $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
262 mount_after_interval_sub $ost_interval $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
264 if [ $rc -ne 0 ]; then
272 # Mount the shared target on the failover server
273 # during unmounting it on the primary server.
274 mount_during_unmount() {
280 local failover_facet=$(get_failover_facet $facet)
286 log "Mounting $device on $facet..."
287 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
289 log "Unmounting $device on $facet..."
293 log "Mounting $device on $failover_facet..."
294 start $failover_facet $device $mnt_opts
295 mount_rc=${PIPESTATUS[0]}
298 unmount_rc=${PIPESTATUS[0]}
300 if [ $mount_rc -eq 0 ]; then
301 error_noexit "mount during unmount of the first filesystem should fail"
302 stop $failover_facet || return ${PIPESTATUS[0]}
306 if [ $unmount_rc -ne 0 ]; then
307 error_noexit "unmount the $device on $facet should succeed"
314 # Mount the shared target on the failover server
315 # after clean unmounting it on the primary server.
316 mount_after_unmount() {
322 local failover_facet=$(get_failover_facet $facet)
324 log "Mounting $device on $facet..."
325 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
327 log "Unmounting $device on $facet..."
328 stop $facet || return ${PIPESTATUS[0]}
330 log "Mounting $device on $failover_facet..."
331 start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
336 # Mount the shared target on the failover server after rebooting
337 # the primary server.
338 mount_after_reboot() {
344 local failover_facet=$(get_failover_facet $facet)
347 log "Mounting $device on $facet..."
348 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
350 if [ "$FAILURE_MODE" = "HARD" ]; then
351 shutdown_facet $facet
353 wait_for_facet $facet
355 replay_barrier_nodf $facet
358 log "Mounting $device on $failover_facet..."
359 start $failover_facet $device $mnt_opts
361 if [ $rc -ne 0 ]; then
362 error_noexit "mount $device on $failover_facet should succeed"
363 stop $facet || return ${PIPESTATUS[0]}
370 # Run e2fsck on the Lustre server target.
378 log "Running e2fsck on the device $device on $facet..."
379 do_facet $facet "$E2FSCK $opts $device"
380 return ${PIPESTATUS[0]}
383 # Check whether there are failover pairs for MDS and OSS servers.
384 check_failover_pair() {
385 [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
386 && { skip_env "failover pair is needed" && return 1; }
392 # Test 1 - two mounts at the same time.
394 check_failover_pair || return 0
396 mount_after_interval 0 0 || return ${PIPESTATUS[0]}
397 stop_services primary || return ${PIPESTATUS[0]}
399 run_test 1 "two mounts at the same time"
401 # Test 2 - one mount delayed by mmp update interval.
403 check_failover_pair || return 0
405 local mdt_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
406 local ost_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)
408 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
409 stop_services primary || return ${PIPESTATUS[0]}
411 run_test 2 "one mount delayed by mmp update interval"
413 # Test 3 - one mount delayed by 2x mmp check interval.
415 check_failover_pair || return 0
417 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
418 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
420 mdt_interval=$((2 * $mdt_interval + 1))
421 ost_interval=$((2 * $ost_interval + 1))
423 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
424 stop_services primary || return ${PIPESTATUS[0]}
426 run_test 3 "one mount delayed by 2x mmp check interval"
428 # Test 4 - one mount delayed by > 2x mmp check interval.
430 check_failover_pair || return 0
432 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
433 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
435 mdt_interval=$((4 * $mdt_interval))
436 ost_interval=$((4 * $ost_interval))
438 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
439 stop_services primary || return ${PIPESTATUS[0]}
441 run_test 4 "one mount delayed by > 2x mmp check interval"
443 # Test 5 - mount during unmount of the first filesystem.
446 check_failover_pair || return 0
448 mount_during_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
449 return ${PIPESTATUS[0]}
452 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
453 mount_during_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
455 if [ $rc -ne 0 ]; then
456 stop $MMP_MDS || return ${PIPESTATUS[0]}
460 stop $MMP_MDS || return ${PIPESTATUS[0]}
462 run_test 5 "mount during unmount of the first filesystem"
464 # Test 6 - mount after clean unmount.
467 check_failover_pair || return 0
469 mount_after_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
470 return ${PIPESTATUS[0]}
473 mount_after_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
475 if [ $rc -ne 0 ]; then
476 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
480 stop_services failover || return ${PIPESTATUS[0]}
482 run_test 6 "mount after clean unmount"
484 # Test 7 - mount after reboot.
487 check_failover_pair || return 0
489 mount_after_reboot $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
490 return ${PIPESTATUS[0]}
493 mount_after_reboot $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
495 if [ $rc -ne 0 ]; then
496 stop $MMP_MDS || return ${PIPESTATUS[0]}
497 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
501 stop_services failover || return ${PIPESTATUS[0]}
502 stop_services primary || return ${PIPESTATUS[0]}
504 run_test 7 "mount after reboot"
506 # Test 8 - mount during e2fsck (should never succeed).
510 run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
514 log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
515 if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
516 error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
517 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
524 run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
528 log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
529 if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
530 error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
531 stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
538 run_test 8 "mount during e2fsck"
540 # Test 9 - mount after aborted e2fsck (should never succeed).
543 local mdt_mmp_check_interval
544 local ost_mmp_check_interval
546 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
547 if ! start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
548 local rc=${PIPESTATUS[0]}
549 stop $MMP_MDS || return ${PIPESTATUS[0]}
552 stop_services primary || return ${PIPESTATUS[0]}
554 mdt_mmp_check_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
555 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fy" &
557 sleep $((2 * $mdt_mmp_check_interval + 1))
558 kill -s ABRT $e2fsck_pid
560 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
561 if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
562 error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"
563 stop $MMP_MDS || return ${PIPESTATUS[0]}
567 reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
570 ost_mmp_check_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
571 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fy" &
573 sleep $((2 * $ost_mmp_check_interval + 1))
574 kill -s ABRT $e2fsck_pid
576 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
577 if start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
578 error_noexit "mount $MMP_OSTDEV on $MMP_OSS should fail"
579 stop $MMP_OSS || return ${PIPESTATUS[0]}
583 reset_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
586 run_test 9 "mount after aborted e2fsck"
588 # Test 10 - e2fsck with mounted filesystem.
592 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
593 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
595 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fn"
597 if [ $rc -ne 8 ]; then
598 error_noexit "e2fsck $MMP_MDSDEV on $MMP_MDS_FAILOVER should return 8"
599 stop $MMP_MDS || return ${PIPESTATUS[0]}
600 [ $rc -ne 0 ] && return $rc || return 1
603 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
604 start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS
606 if [ $rc -ne 0 ]; then
607 stop $MMP_MDS || return ${PIPESTATUS[0]}
611 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fn"
613 if [ $rc -ne 8 ]; then
614 error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER should return 8"
615 stop_services primary || return ${PIPESTATUS[0]}
616 [ $rc -ne 0 ] && return $rc || return 1
619 stop_services primary || return ${PIPESTATUS[0]}
622 run_test 10 "e2fsck with mounted filesystem"
625 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
627 equals_msg $(basename $0): test complete
628 $MMP_RESTORE_MOUNT && setupall
629 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && \
630 grep -q FAIL $TESTSUITELOG && exit 1 || true