2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 # Tests for multiple mount protection (MMP) feature.
7 # Run select tests by setting ONLY, or as arguments to the script.
8 # Skip specific tests by setting EXCEPT.
10 # e.g. ONLY="5 6" or ONLY="`seq 8 11`" or EXCEPT="7"
15 # bug number for skipped test:
16 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$MMP_EXCEPT"}
17 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
19 SRCDIR=$(cd $(dirname $0); echo $PWD)
20 export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH:/sbin
22 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
23 . $LUSTRE/tests/test-framework.sh
25 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
28 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
29 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
30 [ "$MDSFSTYPE" != "ldiskfs" ] && skip "MDS not running ldiskfs" && exit 0
31 [ "$OSTFSTYPE" != "ldiskfs" ] && skip "OST not running ldiskfs" && exit 0
33 # unmount and cleanup the Lustre filesystem
34 MMP_RESTORE_MOUNT=false
35 if is_mounted $MOUNT || is_mounted $MOUNT2; then
37 MMP_RESTORE_MOUNT=true
40 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
45 # Get the failover facet.
46 get_failover_facet() {
48 local failover_facet=${facet}failover
50 local host=$(facet_host $facet)
51 local failover_host=$(facet_host $failover_facet)
53 [ -z "$failover_host" -o "$host" = "$failover_host" ] && \
59 # Initiate the variables for Lustre servers and targets.
61 MMP_MDS=${MMP_MDS:-$SINGLEMDS}
62 MMP_MDS_FAILOVER=$(get_failover_facet $MMP_MDS)
64 local mds_num=$(echo $MMP_MDS | tr -d "mds")
65 MMP_MDSDEV=$(mdsdevname $mds_num)
67 MMP_OSS=${MMP_OSS:-ost1}
68 MMP_OSS_FAILOVER=$(get_failover_facet $MMP_OSS)
70 local oss_num=$(echo $MMP_OSS | tr -d "ost")
71 MMP_OSTDEV=$(ostdevname $oss_num)
74 # Stop the MDS and OSS services on the primary or failover servers.
82 if [ "$flavor" = "failover" ]; then
83 mds_facet=$MMP_MDS_FAILOVER
84 oss_facet=$MMP_OSS_FAILOVER
90 stop $mds_facet $opts || return ${PIPESTATUS[0]}
91 stop $oss_facet $opts || return ${PIPESTATUS[0]}
94 # Enable the MMP feature.
99 do_facet $facet "$TUNE2FS -O mmp $device"
100 return ${PIPESTATUS[0]}
103 # Disable the MMP feature.
108 do_facet $facet "$TUNE2FS -O ^mmp $device"
109 return ${PIPESTATUS[0]}
112 # Set the MMP block to 'fsck' state
117 do_facet $facet "$LUSTRE/tests/mmp_mark.sh $device"
118 return ${PIPESTATUS[0]}
121 # Reset the MMP block (if any) back to the clean state.
126 do_facet $facet "$TUNE2FS -f -E clear-mmp $device"
127 return ${PIPESTATUS[0]}
130 # Check whether the MMP feature is enabled or not.
135 do_facet $facet "$DUMPE2FS -h $device | grep mmp"
136 return ${PIPESTATUS[0]}
139 # Get MMP update interval (in seconds) from the Lustre server target.
140 get_mmp_update_interval() {
145 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
146 | grep 'MMP Update Interval' | cut -d' ' -f4")
147 [ -z "$interval" ] && interval=1
152 # Get MMP check interval (in seconds) from the Lustre server target.
153 get_mmp_check_interval() {
158 interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \
159 | grep 'MMP Check Interval' | cut -d' ' -f4")
160 [ -z "$interval" ] && interval=5
165 # Enable the MMP feature on the Lustre server targets.
169 # The MMP feature is automatically enabled by mkfs.lustre for
170 # new file system at format time if failover is being used.
171 # Otherwise, the Lustre administrator has to manually enable
172 # this feature when the file system is unmounted.
174 local var=${MMP_MDS}failover_HOST
175 if [ -z "${!var}" ]; then
176 log "Failover is not used on MDS, enabling MMP manually..."
177 enable_mmp $MMP_MDS $MMP_MDSDEV || \
178 error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS"
181 var=${MMP_OSS}failover_HOST
182 if [ -z "${!var}" ]; then
183 log "Failover is not used on OSS, enabling MMP manually..."
184 enable_mmp $MMP_OSS $MMP_OSTDEV || \
185 error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS"
188 # check whether the MMP feature is enabled or not
189 mmp_is_enabled $MMP_MDS $MMP_MDSDEV || \
190 error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS"
192 mmp_is_enabled $MMP_OSS $MMP_OSTDEV || \
193 error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS"
196 # Disable the MMP feature on the Lustre server targets
197 # which did not use failover.
200 local var=${MMP_MDS}failover_HOST
201 if [ -z "${!var}" ]; then
202 log "Failover is not used on MDS, disabling MMP manually..."
203 disable_mmp $MMP_MDS $MMP_MDSDEV || \
204 error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS"
205 mmp_is_enabled $MMP_MDS $MMP_MDSDEV && \
206 error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS"
209 var=${MMP_OSS}failover_HOST
210 if [ -z "${!var}" ]; then
211 log "Failover is not used on OSS, disabling MMP manually..."
212 disable_mmp $MMP_OSS $MMP_OSTDEV || \
213 error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS"
214 mmp_is_enabled $MMP_OSS $MMP_OSTDEV && \
215 error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS"
221 # Mount the shared target on the failover server after some interval it's
222 # mounted on the primary server.
223 mount_after_interval_sub() {
231 local failover_facet=$(get_failover_facet $facet)
234 local first_mount_rc=0
235 local second_mount_rc=0
237 log "Mounting $device on $facet..."
238 start $facet $device $opts &
241 if [ $interval -ne 0 ]; then
242 log "sleep $interval..."
246 log "Mounting $device on $failover_facet..."
247 start $failover_facet $device $opts
248 second_mount_rc=${PIPESTATUS[0]}
251 first_mount_rc=${PIPESTATUS[0]}
253 if [ $second_mount_rc -eq 0 -a $first_mount_rc -eq 0 ]; then
254 error_noexit "one mount delayed by mmp interval $interval should fail"
255 stop $facet || return ${PIPESTATUS[0]}
256 [ "$failover_facet" != "$facet" ] && stop $failover_facet || \
257 return ${PIPESTATUS[0]}
259 elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
260 error_noexit "failed to mount on the failover pair $facet,$failover_facet"
261 return $first_mount_rc
267 mount_after_interval() {
268 local mdt_interval=$1
269 local ost_interval=$2
272 mount_after_interval_sub $mdt_interval $MMP_MDSDEV $MMP_MDS \
273 $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
276 mount_after_interval_sub $ost_interval $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
278 if [ $rc -ne 0 ]; then
286 # Mount the shared target on the failover server
287 # during unmounting it on the primary server.
288 mount_during_unmount() {
294 local failover_facet=$(get_failover_facet $facet)
300 log "Mounting $device on $facet..."
301 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
303 log "Unmounting $device on $facet..."
307 log "Mounting $device on $failover_facet..."
308 start $failover_facet $device $mnt_opts
309 mount_rc=${PIPESTATUS[0]}
312 unmount_rc=${PIPESTATUS[0]}
314 if [ $mount_rc -eq 0 ]; then
315 error_noexit "mount during unmount of the first filesystem should fail"
316 stop $failover_facet || return ${PIPESTATUS[0]}
320 if [ $unmount_rc -ne 0 ]; then
321 error_noexit "unmount the $device on $facet should succeed"
328 # Mount the shared target on the failover server
329 # after clean unmounting it on the primary server.
330 mount_after_unmount() {
336 local failover_facet=$(get_failover_facet $facet)
338 log "Mounting $device on $facet..."
339 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
341 log "Unmounting $device on $facet..."
342 stop $facet || return ${PIPESTATUS[0]}
344 log "Mounting $device on $failover_facet..."
345 start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
350 # Mount the shared target on the failover server after rebooting
351 # the primary server.
352 mount_after_reboot() {
358 local failover_facet=$(get_failover_facet $facet)
361 log "Mounting $device on $facet..."
362 start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
364 if [ "$FAILURE_MODE" = "HARD" ]; then
365 shutdown_facet $facet
367 wait_for_facet $facet
369 replay_barrier_nodf $facet
372 log "Mounting $device on $failover_facet..."
373 start $failover_facet $device $mnt_opts
375 if [ $rc -ne 0 ]; then
376 error_noexit "mount $device on $failover_facet should succeed"
377 stop $facet || return ${PIPESTATUS[0]}
384 # Run e2fsck on the Lustre server target.
392 log "Running e2fsck on the device $device on $facet..."
393 do_facet $facet "$E2FSCK $opts $device"
394 return ${PIPESTATUS[0]}
397 # Check whether there are failover pairs for MDS and OSS servers.
398 check_failover_pair() {
399 [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
400 && { skip_env "failover pair is needed" && return 1; }
406 # Test 1 - two mounts at the same time.
408 check_failover_pair || return 0
410 mount_after_interval 0 0 || return ${PIPESTATUS[0]}
411 stop_services primary || return ${PIPESTATUS[0]}
413 run_test 1 "two mounts at the same time"
415 # Test 2 - one mount delayed by mmp update interval.
417 check_failover_pair || return 0
419 local mdt_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
420 local ost_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)
422 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
423 stop_services primary || return ${PIPESTATUS[0]}
425 run_test 2 "one mount delayed by mmp update interval"
427 # Test 3 - one mount delayed by 2x mmp check interval.
429 check_failover_pair || return 0
431 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
432 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
434 mdt_interval=$((2 * $mdt_interval + 1))
435 ost_interval=$((2 * $ost_interval + 1))
437 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
438 stop_services primary || return ${PIPESTATUS[0]}
440 run_test 3 "one mount delayed by 2x mmp check interval"
442 # Test 4 - one mount delayed by > 2x mmp check interval.
444 check_failover_pair || return 0
446 local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
447 local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)
449 mdt_interval=$((4 * $mdt_interval))
450 ost_interval=$((4 * $ost_interval))
452 mount_after_interval $mdt_interval $ost_interval || return ${PIPESTATUS[0]}
453 stop_services primary || return ${PIPESTATUS[0]}
455 run_test 4 "one mount delayed by > 2x mmp check interval"
457 # Test 5 - mount during unmount of the first filesystem.
460 check_failover_pair || return 0
462 mount_during_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
463 return ${PIPESTATUS[0]}
466 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
467 mount_during_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
469 if [ $rc -ne 0 ]; then
470 stop $MMP_MDS || return ${PIPESTATUS[0]}
474 stop $MMP_MDS || return ${PIPESTATUS[0]}
476 run_test 5 "mount during unmount of the first filesystem"
478 # Test 6 - mount after clean unmount.
481 check_failover_pair || return 0
483 mount_after_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
484 return ${PIPESTATUS[0]}
487 mount_after_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
489 if [ $rc -ne 0 ]; then
490 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
494 stop_services failover || return ${PIPESTATUS[0]}
496 run_test 6 "mount after clean unmount"
498 # Test 7 - mount after reboot.
501 check_failover_pair || return 0
503 mount_after_reboot $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS || \
504 return ${PIPESTATUS[0]}
507 mount_after_reboot $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
509 if [ $rc -ne 0 ]; then
510 stop $MMP_MDS || return ${PIPESTATUS[0]}
511 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
515 stop_services failover || return ${PIPESTATUS[0]}
516 stop_services primary || return ${PIPESTATUS[0]}
518 run_test 7 "mount after reboot"
520 # Test 8 - mount during e2fsck (should never succeed).
524 run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
528 log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
529 if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
530 error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
531 stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
538 run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
542 log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
543 if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
544 error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
545 stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
552 run_test 8 "mount during e2fsck"
554 # Test 9 - mount after aborted e2fsck (should never succeed).
556 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
557 if ! start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
558 local rc=${PIPESTATUS[0]}
559 stop $MMP_MDS || return ${PIPESTATUS[0]}
562 stop_services primary || return ${PIPESTATUS[0]}
564 mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
566 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
567 if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
568 error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"
569 stop $MMP_MDS || return ${PIPESTATUS[0]}
573 reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
575 mark_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
577 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
578 if start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
579 error_noexit "mount $MMP_OSTDEV on $MMP_OSS should fail"
580 stop $MMP_OSS || return ${PIPESTATUS[0]}
584 reset_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
587 run_test 9 "mount after aborted e2fsck"
589 # Test 10 - e2fsck with mounted filesystem.
593 log "Mounting $MMP_MDSDEV on $MMP_MDS..."
594 start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
596 run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fn"
599 # e2fsck is called with -n option (Open the filesystem read-only), so
600 # 0 (No errors) and 4 (File system errors left uncorrected) are the only
601 # acceptable exit codes in this case
602 if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
603 error_noexit "e2fsck $MMP_MDSDEV on $MMP_MDS_FAILOVER returned $rc"
604 stop $MMP_MDS || return ${PIPESTATUS[0]}
608 log "Mounting $MMP_OSTDEV on $MMP_OSS..."
609 start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS
611 if [ $rc -ne 0 ]; then
612 stop $MMP_MDS || return ${PIPESTATUS[0]}
616 run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fn"
618 if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
619 error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER returned $rc"
622 stop_services primary || return ${PIPESTATUS[0]}
625 run_test 10 "e2fsck with mounted filesystem"
628 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
630 complete $(basename $0) $SECONDS
631 $MMP_RESTORE_MOUNT && setupall