X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fmmp.sh;h=db5ce5ae08e63667249c51be398ee031677105f5;hb=b78fb445555916e380b1661546c821df14098596;hp=c2ee0fd21c5beba3ebd58c39fbd6475803b6b492;hpb=c840ed6d879c9958f08542cc20ae849bda1f617c;p=fs%2Flustre-release.git diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh index c2ee0fd..db5ce5a 100755 --- a/lustre/tests/mmp.sh +++ b/lustre/tests/mmp.sh @@ -1,5 +1,6 @@ #!/bin/bash -# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*- +# vim:shiftwidth=4:softtabstop=4:tabstop=4: # # Tests for multiple mount protection (MMP) feature. # @@ -135,87 +136,113 @@ mmp_is_enabled() { # Get MMP update interval (in seconds) from the Lustre server target. get_mmp_update_interval() { - local facet=$1 - local device=$2 - local interval - - interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \ - | grep 'MMP Update Interval' | cut -d' ' -f4") - [ -z "$interval" ] && interval=1 - - echo $interval + local facet=$1 + local device=$2 + local interval + + interval=$(do_facet $facet \ + "$DEBUGFS -c -R dump_mmp $device 2>$TMP/mmp.debugfs.msg" | + awk 'tolower($0) ~ /update.interval/ { print $NF }') + [ -z "$interval" ] && interval=5 && + do_facet $facet cat $TMP/mmp.debugfs.msg && + echo "$facet:$device: assume update interval=$interval" 1>&2 || + echo "$facet:$device: got actual update interval=$interval" 1>&2 + + echo $interval } # Get MMP check interval (in seconds) from the Lustre server target. get_mmp_check_interval() { - local facet=$1 - local device=$2 - local interval + local facet=$1 + local device=$2 + local interval + + interval=$(do_facet $facet \ + "$DEBUGFS -c -R dump_mmp $device 2>$TMP/mmp.debugfs.msg" | + awk 'tolower($0) ~ /check.interval/ { print $NF }') + [ -z "$interval" ] && interval=5 && + do_facet $facet cat $TMP/mmp.debugfs.msg && + echo "$facet:$device: assume check interval=$interval" 1>&2 || + echo "$facet:$device: got actual check interval=$interval" 1>&2 + + echo $interval +} - interval=$(do_facet $facet "$DEBUGFS -c -R dump_mmp $device 2>/dev/null \ - | grep 'MMP Check Interval' | cut -d' ' -f4") - [ -z "$interval" ] && interval=5 +# Adjust the MMP update interval (in seconds) on the Lustre server target. +# Specifying an interval of 0 means to use the default interval. +set_mmp_update_interval() { + local facet=$1 + local device=$2 + local interval=${3:-0} - echo $interval + do_facet $facet "$TUNE2FS -E mmp_update_interval=$interval $device" + return ${PIPESTATUS[0]} } +I_ENABLED_MDS=0 +I_ENABLED_OSS=0 + # Enable the MMP feature on the Lustre server targets. mmp_init() { - init_vars - - # The MMP feature is automatically enabled by mkfs.lustre for - # new file system at format time if failover is being used. - # Otherwise, the Lustre administrator has to manually enable - # this feature when the file system is unmounted. - - local var=${MMP_MDS}failover_HOST - if [ -z "${!var}" ]; then - log "Failover is not used on MDS, enabling MMP manually..." - enable_mmp $MMP_MDS $MMP_MDSDEV || \ - error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS" - fi - - var=${MMP_OSS}failover_HOST - if [ -z "${!var}" ]; then - log "Failover is not used on OSS, enabling MMP manually..." - enable_mmp $MMP_OSS $MMP_OSTDEV || \ - error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS" - fi - - # check whether the MMP feature is enabled or not - mmp_is_enabled $MMP_MDS $MMP_MDSDEV || \ - error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS" - - mmp_is_enabled $MMP_OSS $MMP_OSTDEV || \ - error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS" + init_vars + + if [ $(facet_fstype $MMP_MDS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based MDTs" + exit + fi + + if [ $(facet_fstype $MMP_OSS) != ldiskfs ]; then + skip "Only applicable to ldiskfs-based OSTs" + exit + fi + + mmp_is_enabled $MMP_MDS $MMP_MDSDEV || + { + log "MMP is not enabled on MDS, enabling it manually..." + enable_mmp $MMP_MDS $MMP_MDSDEV || + error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS" + I_ENABLED_MDS=1 + } + + mmp_is_enabled $MMP_OSS $MMP_OSTDEV || + { + log "MMP is not enabled on OSS, enabling it manually..." + enable_mmp $MMP_OSS $MMP_OSTDEV || + error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS" + I_ENABLED_OSS=1 + } + + # check whether the MMP feature is enabled or not + mmp_is_enabled $MMP_MDS $MMP_MDSDEV || + error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS" + + mmp_is_enabled $MMP_OSS $MMP_OSTDEV || + error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS" } # Disable the MMP feature on the Lustre server targets -# which did not use failover. mmp_fini() { - local var=${MMP_MDS}failover_HOST - if [ -z "${!var}" ]; then - log "Failover is not used on MDS, disabling MMP manually..." - disable_mmp $MMP_MDS $MMP_MDSDEV || \ - error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS" - mmp_is_enabled $MMP_MDS $MMP_MDSDEV && \ - error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS" - fi - - var=${MMP_OSS}failover_HOST - if [ -z "${!var}" ]; then - log "Failover is not used on OSS, disabling MMP manually..." - disable_mmp $MMP_OSS $MMP_OSTDEV || \ - error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS" - mmp_is_enabled $MMP_OSS $MMP_OSTDEV && \ - error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS" - fi - - return 0 + if [ $I_ENABLED_MDS -eq 1 ]; then + log "Disabling MMP on $MMP_MDSDEV on $MMP_MDS manually..." + disable_mmp $MMP_MDS $MMP_MDSDEV || + error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS" + mmp_is_enabled $MMP_MDS $MMP_MDSDEV && + error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS" + fi + + if [ $I_ENABLED_OSS -eq 1 ]; then + log "Disabling MMP on $MMP_OSTDEV on $MMP_OSS manually..." + disable_mmp $MMP_OSS $MMP_OSTDEV || + error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS" + mmp_is_enabled $MMP_OSS $MMP_OSTDEV && + error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS" + fi + + return 0 } -# Mount the shared target on the failover server after some interval it's +# Mount the shared target on the failover server after some interval it's # mounted on the primary server. mount_after_interval_sub() { local interval=$1 @@ -254,7 +281,7 @@ mount_after_interval_sub() { return ${PIPESTATUS[0]} return 1 elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then - error_noexit "failed to mount on the failover pair $facet,$failover_facet" + error_noexit "mount failure on failover pair $facet,$failover_facet" return $first_mount_rc fi @@ -280,7 +307,7 @@ mount_after_interval() { return 0 } -# Mount the shared target on the failover server +# Mount the shared target on the failover server # during unmounting it on the primary server. mount_during_unmount() { local device=$1 @@ -322,7 +349,7 @@ mount_during_unmount() { return 0 } -# Mount the shared target on the failover server +# Mount the shared target on the failover server # after clean unmounting it on the primary server. mount_after_unmount() { local device=$1 @@ -336,7 +363,7 @@ mount_after_unmount() { start $facet $device $mnt_opts || return ${PIPESTATUS[0]} log "Unmounting $device on $facet..." - stop $facet || return ${PIPESTATUS[0]} + stop $facet || return ${PIPESTATUS[0]} log "Mounting $device on $failover_facet..." start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]} @@ -380,15 +407,15 @@ mount_after_reboot() { # Run e2fsck on the Lustre server target. run_e2fsck() { - local facet=$1 - shift - local device=$1 - shift - local opts="$@" - - log "Running e2fsck on the device $device on $facet..." - do_facet $facet "$E2FSCK $opts $device" - return ${PIPESTATUS[0]} + local facet=$1 + shift + local device=$1 + shift + local opts="$@" + + echo "Running e2fsck on the device $device on $facet..." + do_facet $facet "$E2FSCK $opts $device" + return ${PIPESTATUS[0]} } # Check whether there are failover pairs for MDS and OSS servers. @@ -516,35 +543,54 @@ run_test 7 "mount after reboot" # Test 8 - mount during e2fsck (should never succeed). test_8() { - local e2fsck_pid - - run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" & - e2fsck_pid=$! - sleep 1 - - log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..." - if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then - error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail" - stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]} - return 1 - fi - - wait $e2fsck_pid - - echo - run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" & - e2fsck_pid=$! - sleep 1 - - log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..." - if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then - error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail" - stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]} - return 2 - fi - - wait $e2fsck_pid - return 0 + local e2fsck_pid + local saved_interval + local new_interval + + # After writing a new sequence number into the MMP block, e2fsck will + # sleep at least (2 * new_interval + 1) seconds before it goes into + # e2fsck passes. + new_interval=30 + + # MDT + saved_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV) + set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $new_interval + + run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" & + e2fsck_pid=$! + sleep 5 + + if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then + error_noexit \ + "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail" + stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]} + set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $saved_interval + return 1 + fi + + wait $e2fsck_pid + set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $saved_interval + + # OST + echo + saved_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV) + set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $new_interval + + run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" & + e2fsck_pid=$! + sleep 5 + + if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then + error_noexit \ + "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail" + stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]} + set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $saved_interval + return 2 + fi + + wait $e2fsck_pid + set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $saved_interval + return 0 } run_test 8 "mount during e2fsck" @@ -559,7 +605,7 @@ test_9() { stop_services primary || return ${PIPESTATUS[0]} mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]} - + log "Mounting $MMP_MDSDEV on $MMP_MDS..." if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail" @@ -567,8 +613,6 @@ test_9() { return 1 fi - kill -9 $debugfspid - reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]} mark_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]} @@ -626,6 +670,6 @@ run_test 10 "e2fsck with mounted filesystem" mmp_fini FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR -complete $(basename $0) $SECONDS +complete $SECONDS $MMP_RESTORE_MOUNT && setupall exit_status