2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 # Tests for multiple mount protection (MMP) feature.
6 # Run select tests by setting ONLY, or as arguments to the script.
7 # Skip specific tests by setting EXCEPT.
9 # e.g. ONLY="5 6" or ONLY="`seq 8 11`" or EXCEPT="7"
14 # bug number for skipped test:
15 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$MMP_EXCEPT"}
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 SRCDIR=$(cd $(dirname $0); echo $PWD)
19 export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH:/sbin
21 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
22 . $LUSTRE/tests/test-framework.sh
24 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
26 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
27 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
29 # unmount and cleanup the Lustre filesystem
30 MMP_RESTORE_MOUNT=false
31 if is_mounted $MOUNT || is_mounted $MOUNT2; then
33 MMP_RESTORE_MOUNT=true
36 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
41 # Get the failover facet.
42 get_failover_facet() {
44 local failover_facet=${facet}failover
46 local host=$(facet_host $facet)
47 local failover_host=$(facet_host $failover_facet)
49 [ -z "$failover_host" -o "$host" = "$failover_host" ] && \
55 # Initiate the variables for Lustre servers and targets.
58 MMP_MDS_FAILOVER=$(get_failover_facet $MMP_MDS)
62 MMP_OSS=${MMP_OSS:-ost1}
63 MMP_OSS_FAILOVER=$(get_failover_facet $MMP_OSS)
65 local oss_num=$(echo $MMP_OSS | tr -d "ost")
66 MMP_OSTDEV=$(ostdevname $oss_num)
69 # Stop the MDS and OSS services on the primary or failover servers.
77 if [ "$flavor" = "failover" ]; then
78 mds_facet=$MMP_MDS_FAILOVER
79 oss_facet=$MMP_OSS_FAILOVER
85 stop $mds_facet $opts || return ${PIPESTATUS[0]}
86 stop $oss_facet $opts || return ${PIPESTATUS[0]}
89 # Enable the MMP feature.
94 do_facet $facet "$TUNE2FS -O mmp $device"
95 return ${PIPESTATUS[0]}
98 # Disable the MMP feature.
103 do_facet $facet "$TUNE2FS -O ^mmp $device"
104 return ${PIPESTATUS[0]}
107 # Reset the MMP block (if any) back to the clean state.
112 do_facet $facet "$TUNE2FS -f -E clear-mmp $device"
113 return ${PIPESTATUS[0]}
116 # Check whether the MMP feature is enabled or not.
121 do_facet $facet "$DUMPE2FS -h $device | grep mmp"
122 return ${PIPESTATUS[0]}
125 # Get MMP update interval (in seconds) from the Lustre server target.
126 get_mmp_update_interval() {
131 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
132 | grep 'MMP Update Interval' | cut -d' ' -f4")
133 [ -z "$interval" ] && interval=1
138 # Get MMP check interval (in seconds) from the Lustre server target.
139 get_mmp_check_interval() {
144 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
145 | grep 'MMP Check Interval' | cut -d' ' -f4")
146 [ -z "$interval" ] && interval=5
151 # Enable the MMP feature on the Lustre server targets.
155 # The MMP feature is automatically enabled by mkfs.lustre for
156 # new file system at format time if failover is being used.
157 # Otherwise, the Lustre administrator has to manually enable
158 # this feature when the file system is unmounted.
160 if [ -z "$mdsfailover_HOST" ]; then
161 log "Failover is not used on MDS, enabling MMP manually..."
162 enable_mmp $MMP_MDS $MMP_MDSDEV || \
163 error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS"
166 if [ -z "$ostfailover_HOST" ]; then
167 log "Failover is not used on OSS, enabling MMP manually..."
168 enable_mmp $MMP_OSS $MMP_OSTDEV || \
169 error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS"
172 # check whether the MMP feature is enabled or not
173 mmp_is_enabled $MMP_MDS $MMP_MDSDEV || \
174 error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS"
176 mmp_is_enabled $MMP_OSS $MMP_OSTDEV || \
177 error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS"
180 # Disable the MMP feature on the Lustre server targets
181 # which did not use failover.
184 if [ -z "$mdsfailover_HOST" ]; then
185 log "Failover is not used on MDS, disabling MMP manually..."
186 disable_mmp $MMP_MDS $MMP_MDSDEV || \
187 error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS"
188 mmp_is_enabled $MMP_MDS $MMP_MDSDEV && \
189 error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS"
192 if [ -z "$ostfailover_HOST" ]; then
193 log "Failover is not used on OSS, disabling MMP manually..."
194 disable_mmp $MMP_OSS $MMP_OSTDEV || \
195 error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS"
196 mmp_is_enabled $MMP_OSS $MMP_OSTDEV && \
197 error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS"
203 # Mount the shared target on the failover server after some interval it's
204 # mounted on the primary server.
205 mount_after_interval_sub() {
213 local failover_facet=$(get_failover_facet $facet)
216 local first_mount_rc=0
217 local second_mount_rc=0
219 log "Mounting $device on $facet..."
220 start $facet $device $opts &
223 if [ $interval -ne 0 ]; then
224 log "sleep $interval..."
228 log "Mounting $device on $failover_facet..."
229 start $failover_facet $device $opts
230 second_mount_rc=${PIPESTATUS[0]}
233 first_mount_rc=${PIPESTATUS[0]}
235 if [ $second_mount_rc -eq 0 -a $first_mount_rc -eq 0 ]; then
236 error_noexit "one mount delayed by mmp interval $interval should fail"
237 stop $facet || return ${PIPESTATUS[0]}
238 [ "$failover_facet" != "$facet" ] && stop $failover_facet || \
239 return ${PIPESTATUS[0]}
241 elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
242 error_noexit "failed to mount on the failover pair $facet,$failover_facet"
243 return $first_mount_rc
249 mount_after_interval() {
250 local mdt_interval=$1
251 local ost_interval=$2
254 mount_after_interval_sub $mdt_interval $MMP_MDSDEV $MMP_MDS \
255 $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
258 mount_after_interval_sub $ost_interval $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
260 if [ $rc -ne 0 ]; then
268 # Mount the shared target on the failover server
269 # during unmounting it on the primary server.
270 mount_during_unmount() {
276 local failover_facet=$(get_failover_facet $facet)
282 log "Mounting $device on $facet..."
283 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
288 start $failover_facet $device $mnt_opts
289 mount_rc=${PIPESTATUS[0]}
292 unmount_rc=${PIPESTATUS[0]}
294 if [ $mount_rc -eq 0 ]; then
295 error_noexit "mount during unmount of the first filesystem should fail"
296 stop $failover_facet || return ${PIPESTATUS[0]}
300 if [ $unmount_rc -ne 0 ]; then
301 error_noexit "unmount the $device on $facet should succeed"
308 # Mount the shared target on the failover server
309 # after clean unmounting it on the primary server.
310 mount_after_unmount() {
316 local failover_facet=$(get_failover_facet $facet)
318 log "Mounting $device on $facet..."
319 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
321 log "Unmounting $device on $facet..."
322 stop $facet || return ${PIPESTATUS[0]}
324 log "Mounting $device on $failover_facet..."
325 start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
330 # Mount the shared target on the failover server after rebooting
331 # the primary server.
332 mount_after_reboot() {
338 local failover_facet=$(get_failover_facet $facet)
341 log "Mounting $device on $facet..."
342 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
344 if [ "$FAILURE_MODE" = "HARD" ]; then
345 shutdown_facet $facet
349 replay_barrier_nodf $facet
352 log "Mounting $device on $failover_facet..."
353 start $failover_facet $device $mnt_opts
355 if [ $rc -ne 0 ]; then
356 error_noexit "mount $device on $failover_facet should succeed"
357 stop $facet || return ${PIPESTATUS[0]}
364 # Run e2fsck on the Lustre server target.
372 log "Running e2fsck on the device $device on $facet..."
373 do_facet $facet "$E2FSCK $opts $device"
374 return ${PIPESTATUS[0]}
377 # Check whether there are failover pairs for MDS and OSS servers.
378 check_failover_pair() {
379 [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
380 && { skip_env "failover pair is needed" && return 1; }
386 # Test 1 - two mounts at the same time.
388 check_failover_pair || return 0
390 mount_after_interval 0 0 || return ${PIPESTATUS[0]}
391 stop_services primary || return ${PIPESTATUS[0]}
393 run_test 1 "two mounts at the same time"
395 # Test 2 - one mount delayed by mmp update interval.
397 check_failover_pair || return 0
399 local mdt_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
400 local ost_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)
402 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
403 stop_services primary || return ${PIPESTATUS[0]}
405 run_test 2 "one mount delayed by mmp update interval"
407 # Test 3 - one mount delayed by 2x mmp check interval.
409 check_failover_pair || return 0
411 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
412 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
414 mdt_interval=$((2 * $mdt_interval + 1))
415 ost_interval=$((2 * $ost_interval + 1))
417 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
418 stop_services primary || return ${PIPESTATUS[0]}
420 run_test 3 "one mount delayed by 2x mmp check interval"
422 # Test 4 - one mount delayed by > 2x mmp check interval.
424 check_failover_pair || return 0
426 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
427 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
429 mdt_interval=$((4 * $mdt_interval))
430 ost_interval=$((4 * $ost_interval))
432 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
433 stop_services primary || return ${PIPESTATUS[0]}
435 run_test 4 "one mount delayed by > 2x mmp check interval"
437 # Test 5 - mount during unmount of the first filesystem.
440 check_failover_pair || return 0
442 mount_during_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
443 return ${PIPESTATUS[0]}
446 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
447 mount_during_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
449 if [ $rc -ne 0 ]; then
450 stop $MMP_MDS || return ${PIPESTATUS[0]}
454 stop $MMP_MDS || return ${PIPESTATUS[0]}
456 run_test 5 "mount during unmount of the first filesystem"
458 # Test 6 - mount after clean unmount.
461 check_failover_pair || return 0
463 mount_after_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
464 return ${PIPESTATUS[0]}
467 mount_after_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
469 if [ $rc -ne 0 ]; then
470 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
474 stop_services failover || return ${PIPESTATUS[0]}
476 run_test 6 "mount after clean unmount"
478 # Test 7 - mount after reboot.
481 check_failover_pair || return 0
483 mount_after_reboot $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
484 return ${PIPESTATUS[0]}
487 mount_after_reboot $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
489 if [ $rc -ne 0 ]; then
490 stop $MMP_MDS || return ${PIPESTATUS[0]}
491 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
495 stop_services failover || return ${PIPESTATUS[0]}
496 stop_services primary || return ${PIPESTATUS[0]}
498 run_test 7 "mount after reboot"
500 # Test 8 - mount during e2fsck (should never succeed).
504 run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
508 log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
509 if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
510 error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
511 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
518 run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
522 log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
523 if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
524 error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
525 stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
532 run_test 8 "mount during e2fsck"
534 # Test 9 - mount after aborted e2fsck (should never succeed).
537 local mdt_mmp_check_interval
538 local ost_mmp_check_interval
540 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
541 if ! start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
542 local rc=${PIPESTATUS[0]}
543 stop $MMP_MDS || return ${PIPESTATUS[0]}
546 stop_services primary || return ${PIPESTATUS[0]}
548 mdt_mmp_check_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
549 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fy" &
551 sleep $((2 * $mdt_mmp_check_interval))
552 kill -s ABRT $e2fsck_pid
554 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
555 if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
556 error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"
557 stop $MMP_MDS || return ${PIPESTATUS[0]}
561 reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
564 ost_mmp_check_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
565 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fy" &
567 sleep $((2 * $ost_mmp_check_interval))
568 kill -s ABRT $e2fsck_pid
570 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
571 if start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
572 error_noexit "mount $MMP_OSTDEV on $MMP_OSS should fail"
573 stop $MMP_OSS || return ${PIPESTATUS[0]}
577 reset_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
580 run_test 9 "mount after aborted e2fsck"
582 # Test 10 - e2fsck with mounted filesystem.
586 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
587 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
589 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fn"
591 if [ $rc -ne 8 ]; then
592 error_noexit "e2fsck $MMP_MDSDEV on $MMP_MDS_FAILOVER should return 8"
593 stop $MMP_MDS || return ${PIPESTATUS[0]}
594 [ $rc -ne 0 ] && return $rc || return 1
597 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
598 start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS
600 if [ $rc -ne 0 ]; then
601 stop $MMP_MDS || return ${PIPESTATUS[0]}
605 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fn"
607 if [ $rc -ne 8 ]; then
608 error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER should return 8"
609 stop_services primary || return ${PIPESTATUS[0]}
610 [ $rc -ne 0 ] && return $rc || return 1
613 stop_services primary || return ${PIPESTATUS[0]}
616 run_test 10 "e2fsck with mounted filesystem"
619 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
621 equals_msg $(basename $0): test complete
622 $MMP_RESTORE_MOUNT && setupall
623 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && \
624 grep -q FAIL $TESTSUITELOG && exit 1 || true