2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 # Tests for multiple mount protection (MMP) feature.
6 # Run select tests by setting ONLY, or as arguments to the script.
7 # Skip specific tests by setting EXCEPT.
9 # e.g. ONLY="5 6" or ONLY="`seq 8 11`" or EXCEPT="7"
14 # bug number for skipped test:
15 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$MMP_EXCEPT"}
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 SRCDIR=$(cd $(dirname $0); echo $PWD)
19 export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH:/sbin
21 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
22 . $LUSTRE/tests/test-framework.sh
24 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
27 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
28 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
29 [ "$MDSFSTYPE" != "ldiskfs" ] && skip "MDS not running ldiskfs" && exit 0
30 [ "$OSTFSTYPE" != "ldiskfs" ] && skip "OST not running ldiskfs" && exit 0
32 # unmount and cleanup the Lustre filesystem
33 MMP_RESTORE_MOUNT=false
34 if is_mounted $MOUNT || is_mounted $MOUNT2; then
36 MMP_RESTORE_MOUNT=true
39 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
44 # Get the failover facet.
45 get_failover_facet() {
47 local failover_facet=${facet}failover
49 local host=$(facet_host $facet)
50 local failover_host=$(facet_host $failover_facet)
52 [ -z "$failover_host" -o "$host" = "$failover_host" ] && \
58 # Initiate the variables for Lustre servers and targets.
60 MMP_MDS=${MMP_MDS:-$SINGLEMDS}
61 MMP_MDS_FAILOVER=$(get_failover_facet $MMP_MDS)
63 local mds_num=$(echo $MMP_MDS | tr -d "mds")
64 MMP_MDSDEV=$(mdsdevname $mds_num)
66 MMP_OSS=${MMP_OSS:-ost1}
67 MMP_OSS_FAILOVER=$(get_failover_facet $MMP_OSS)
69 local oss_num=$(echo $MMP_OSS | tr -d "ost")
70 MMP_OSTDEV=$(ostdevname $oss_num)
73 # Stop the MDS and OSS services on the primary or failover servers.
81 if [ "$flavor" = "failover" ]; then
82 mds_facet=$MMP_MDS_FAILOVER
83 oss_facet=$MMP_OSS_FAILOVER
89 stop $mds_facet $opts || return ${PIPESTATUS[0]}
90 stop $oss_facet $opts || return ${PIPESTATUS[0]}
93 # Enable the MMP feature.
98 do_facet $facet "$TUNE2FS -O mmp $device"
99 return ${PIPESTATUS[0]}
102 # Disable the MMP feature.
107 do_facet $facet "$TUNE2FS -O ^mmp $device"
108 return ${PIPESTATUS[0]}
111 # Set the MMP block to 'fsck' state
116 do_facet $facet "$LUSTRE/tests/mmp_mark.sh $device"
117 return ${PIPESTATUS[0]}
120 # Reset the MMP block (if any) back to the clean state.
125 do_facet $facet "$TUNE2FS -f -E clear-mmp $device"
126 return ${PIPESTATUS[0]}
129 # Check whether the MMP feature is enabled or not.
134 do_facet $facet "$DUMPE2FS -h $device | grep mmp"
135 return ${PIPESTATUS[0]}
138 # Get MMP update interval (in seconds) from the Lustre server target.
139 get_mmp_update_interval() {
144 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
145 | grep 'MMP Update Interval' | cut -d' ' -f4")
146 [ -z "$interval" ] && interval=1
151 # Get MMP check interval (in seconds) from the Lustre server target.
152 get_mmp_check_interval() {
157 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
158 | grep 'MMP Check Interval' | cut -d' ' -f4")
159 [ -z "$interval" ] && interval=5
164 # Enable the MMP feature on the Lustre server targets.
168 # The MMP feature is automatically enabled by mkfs.lustre for
169 # new file system at format time if failover is being used.
170 # Otherwise, the Lustre administrator has to manually enable
171 # this feature when the file system is unmounted.
173 local var=${MMP_MDS}failover_HOST
174 if [ -z "${!var}" ]; then
175 log "Failover is not used on MDS, enabling MMP manually..."
176 enable_mmp $MMP_MDS $MMP_MDSDEV || \
177 error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS"
180 var=${MMP_OSS}failover_HOST
181 if [ -z "${!var}" ]; then
182 log "Failover is not used on OSS, enabling MMP manually..."
183 enable_mmp $MMP_OSS $MMP_OSTDEV || \
184 error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS"
187 # check whether the MMP feature is enabled or not
188 mmp_is_enabled $MMP_MDS $MMP_MDSDEV || \
189 error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS"
191 mmp_is_enabled $MMP_OSS $MMP_OSTDEV || \
192 error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS"
195 # Disable the MMP feature on the Lustre server targets
196 # which did not use failover.
199 local var=${MMP_MDS}failover_HOST
200 if [ -z "${!var}" ]; then
201 log "Failover is not used on MDS, disabling MMP manually..."
202 disable_mmp $MMP_MDS $MMP_MDSDEV || \
203 error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS"
204 mmp_is_enabled $MMP_MDS $MMP_MDSDEV && \
205 error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS"
208 var=${MMP_OSS}failover_HOST
209 if [ -z "${!var}" ]; then
210 log "Failover is not used on OSS, disabling MMP manually..."
211 disable_mmp $MMP_OSS $MMP_OSTDEV || \
212 error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS"
213 mmp_is_enabled $MMP_OSS $MMP_OSTDEV && \
214 error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS"
220 # Mount the shared target on the failover server after some interval it's
221 # mounted on the primary server.
222 mount_after_interval_sub() {
230 local failover_facet=$(get_failover_facet $facet)
233 local first_mount_rc=0
234 local second_mount_rc=0
236 log "Mounting $device on $facet..."
237 start $facet $device $opts &
240 if [ $interval -ne 0 ]; then
241 log "sleep $interval..."
245 log "Mounting $device on $failover_facet..."
246 start $failover_facet $device $opts
247 second_mount_rc=${PIPESTATUS[0]}
250 first_mount_rc=${PIPESTATUS[0]}
252 if [ $second_mount_rc -eq 0 -a $first_mount_rc -eq 0 ]; then
253 error_noexit "one mount delayed by mmp interval $interval should fail"
254 stop $facet || return ${PIPESTATUS[0]}
255 [ "$failover_facet" != "$facet" ] && stop $failover_facet || \
256 return ${PIPESTATUS[0]}
258 elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
259 error_noexit "failed to mount on the failover pair $facet,$failover_facet"
260 return $first_mount_rc
266 mount_after_interval() {
267 local mdt_interval=$1
268 local ost_interval=$2
271 mount_after_interval_sub $mdt_interval $MMP_MDSDEV $MMP_MDS \
272 $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
275 mount_after_interval_sub $ost_interval $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
277 if [ $rc -ne 0 ]; then
285 # Mount the shared target on the failover server
286 # during unmounting it on the primary server.
287 mount_during_unmount() {
293 local failover_facet=$(get_failover_facet $facet)
299 log "Mounting $device on $facet..."
300 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
302 log "Unmounting $device on $facet..."
306 log "Mounting $device on $failover_facet..."
307 start $failover_facet $device $mnt_opts
308 mount_rc=${PIPESTATUS[0]}
311 unmount_rc=${PIPESTATUS[0]}
313 if [ $mount_rc -eq 0 ]; then
314 error_noexit "mount during unmount of the first filesystem should fail"
315 stop $failover_facet || return ${PIPESTATUS[0]}
319 if [ $unmount_rc -ne 0 ]; then
320 error_noexit "unmount the $device on $facet should succeed"
327 # Mount the shared target on the failover server
328 # after clean unmounting it on the primary server.
329 mount_after_unmount() {
335 local failover_facet=$(get_failover_facet $facet)
337 log "Mounting $device on $facet..."
338 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
340 log "Unmounting $device on $facet..."
341 stop $facet || return ${PIPESTATUS[0]}
343 log "Mounting $device on $failover_facet..."
344 start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
349 # Mount the shared target on the failover server after rebooting
350 # the primary server.
351 mount_after_reboot() {
357 local failover_facet=$(get_failover_facet $facet)
360 log "Mounting $device on $facet..."
361 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
363 if [ "$FAILURE_MODE" = "HARD" ]; then
364 shutdown_facet $facet
366 wait_for_facet $facet
368 replay_barrier_nodf $facet
371 log "Mounting $device on $failover_facet..."
372 start $failover_facet $device $mnt_opts
374 if [ $rc -ne 0 ]; then
375 error_noexit "mount $device on $failover_facet should succeed"
376 stop $facet || return ${PIPESTATUS[0]}
383 # Run e2fsck on the Lustre server target.
391 log "Running e2fsck on the device $device on $facet..."
392 do_facet $facet "$E2FSCK $opts $device"
393 return ${PIPESTATUS[0]}
396 # Check whether there are failover pairs for MDS and OSS servers.
397 check_failover_pair() {
398 [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
399 && { skip_env "failover pair is needed" && return 1; }
405 # Test 1 - two mounts at the same time.
407 check_failover_pair || return 0
409 mount_after_interval 0 0 || return ${PIPESTATUS[0]}
410 stop_services primary || return ${PIPESTATUS[0]}
412 run_test 1 "two mounts at the same time"
414 # Test 2 - one mount delayed by mmp update interval.
416 check_failover_pair || return 0
418 local mdt_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
419 local ost_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)
421 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
422 stop_services primary || return ${PIPESTATUS[0]}
424 run_test 2 "one mount delayed by mmp update interval"
426 # Test 3 - one mount delayed by 2x mmp check interval.
428 check_failover_pair || return 0
430 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
431 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
433 mdt_interval=$((2 * $mdt_interval + 1))
434 ost_interval=$((2 * $ost_interval + 1))
436 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
437 stop_services primary || return ${PIPESTATUS[0]}
439 run_test 3 "one mount delayed by 2x mmp check interval"
441 # Test 4 - one mount delayed by > 2x mmp check interval.
443 check_failover_pair || return 0
445 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
446 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
448 mdt_interval=$((4 * $mdt_interval))
449 ost_interval=$((4 * $ost_interval))
451 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
452 stop_services primary || return ${PIPESTATUS[0]}
454 run_test 4 "one mount delayed by > 2x mmp check interval"
456 # Test 5 - mount during unmount of the first filesystem.
459 check_failover_pair || return 0
461 mount_during_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
462 return ${PIPESTATUS[0]}
465 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
466 mount_during_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
468 if [ $rc -ne 0 ]; then
469 stop $MMP_MDS || return ${PIPESTATUS[0]}
473 stop $MMP_MDS || return ${PIPESTATUS[0]}
475 run_test 5 "mount during unmount of the first filesystem"
477 # Test 6 - mount after clean unmount.
480 check_failover_pair || return 0
482 mount_after_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
483 return ${PIPESTATUS[0]}
486 mount_after_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
488 if [ $rc -ne 0 ]; then
489 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
493 stop_services failover || return ${PIPESTATUS[0]}
495 run_test 6 "mount after clean unmount"
497 # Test 7 - mount after reboot.
500 check_failover_pair || return 0
502 mount_after_reboot $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
503 return ${PIPESTATUS[0]}
506 mount_after_reboot $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
508 if [ $rc -ne 0 ]; then
509 stop $MMP_MDS || return ${PIPESTATUS[0]}
510 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
514 stop_services failover || return ${PIPESTATUS[0]}
515 stop_services primary || return ${PIPESTATUS[0]}
517 run_test 7 "mount after reboot"
519 # Test 8 - mount during e2fsck (should never succeed).
523 run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
527 log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
528 if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
529 error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
530 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
537 run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
541 log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
542 if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
543 error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
544 stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
551 run_test 8 "mount during e2fsck"
553 # Test 9 - mount after aborted e2fsck (should never succeed).
555 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
556 if ! start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
557 local rc=${PIPESTATUS[0]}
558 stop $MMP_MDS || return ${PIPESTATUS[0]}
561 stop_services primary || return ${PIPESTATUS[0]}
563 mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
565 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
566 if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
567 error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"
568 stop $MMP_MDS || return ${PIPESTATUS[0]}
572 reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
574 mark_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
576 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
577 if start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
578 error_noexit "mount $MMP_OSTDEV on $MMP_OSS should fail"
579 stop $MMP_OSS || return ${PIPESTATUS[0]}
583 reset_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
586 run_test 9 "mount after aborted e2fsck"
588 # Test 10 - e2fsck with mounted filesystem.
592 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
593 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
595 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fn"
598 # e2fsck is called with -n option (Open the filesystem read-only), so
599 # 0 (No errors) and 4 (File system errors left uncorrected) are the only
600 # acceptable exit codes in this case
601 if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
602 error_noexit "e2fsck $MMP_MDSDEV on $MMP_MDS_FAILOVER returned $rc"
603 stop $MMP_MDS || return ${PIPESTATUS[0]}
607 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
608 start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS
610 if [ $rc -ne 0 ]; then
611 stop $MMP_MDS || return ${PIPESTATUS[0]}
615 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fn"
617 if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
618 error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER returned $rc"
621 stop_services primary || return ${PIPESTATUS[0]}
624 run_test 10 "e2fsck with mounted filesystem"
627 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
629 complete $(basename $0) $SECONDS
630 $MMP_RESTORE_MOUNT && setupall