LU-1689 tests: fix mount during e2fsck test

The current mmp test 8 (mount during e2fsck) has two time issues:
1) the mount operation may start before e2fsck
2) the e2fsck operation may stop before mount

This patch fixes the above issues by providing enough time for e2fsck
operation to be started before mount operation, and setting the
superblock free_blocks_count field with 0 to force e2fsck checking
the Lustre server target device, which provides enough time for
the mount operation to be started during the e2fsck operation.

Test-Parameters: testlist=mmp
Signed-off-by: Minh Diep <mdiep@whamcloud.com>
Signed-off-by: Yu Jian <yujian@whamcloud.com>
Change-Id: I37e07ffdb6669dfabceaab5ef242460abe204ba0
Reviewed-on: http://review.whamcloud.com/3569
Reviewed-by: Li Wei <liwei@whamcloud.com>
Reviewed-by: Chris Gearing <chris.gearing@intel.com>
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index c8dd697..00b32fa 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -28,7 +28,7 @@ noinst_SCRIPTS += lnet-selftest.sh obdfilter-survey.sh mmp.sh mmp_mark.sh
 noinst_SCRIPTS += sgpdd-survey.sh maloo_upload.sh auster setup-nfs.sh
 noinst_SCRIPTS += mds-survey.sh parallel-scale-nfs.sh
 noinst_SCRIPTS += parallel-scale-nfsv3.sh parallel-scale-nfsv4.sh
-noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh
+noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh e2fsck.exp
 nobase_noinst_SCRIPTS = cfg/local.sh
 nobase_noinst_SCRIPTS += test-groups/regression test-groups/regression-mpi
 nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh

diff --git a/lustre/tests/e2fsck.exp b/lustre/tests/e2fsck.exp
new file mode 100755 (executable)
index 0000000..6a41a6c
--- /dev/null
+++ b/lustre/tests/e2fsck.exp
@@ -0,0 +1,28 @@
+#!/usr/bin/expect -f
+#
+# This expect script invokes e2fsck to check the filesystem
+# on a device and allows e2fsck to be run interactively.
+#
+set env(PATH) "/usr/bin:/bin:/usr/sbin:/sbin"
+
+log_user 1
+
+if {$argc == 1} {
+       set dev [lindex $argv 0]
+} else {
+       set prog [lindex [split $argv0 "/"] end]
+       send_user "Usage: $prog <device>\r\n"
+       exit 1
+}
+
+spawn e2fsck -f $dev
+expect {
+       -gl "<y>? " {
+               sleep 30
+               send "yes\r"
+               exp_continue
+       }
+       -gl "FILE SYSTEM WAS MODIFIED" {
+               exit 0
+       }
+}

diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh
index 59fede1..3128691 100755 (executable)
--- a/lustre/tests/mmp.sh
+++ b/lustre/tests/mmp.sh
@@ -226,7 +226,7 @@ mmp_fini() {
     return 0
 }
 
-# Mount the shared target on the failover server after some interval it's 
+# Mount the shared target on the failover server after some interval it's
 # mounted on the primary server.
 mount_after_interval_sub() {
     local interval=$1
@@ -265,7 +265,7 @@ mount_after_interval_sub() {
             return ${PIPESTATUS[0]}
         return 1
     elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
-        error_noexit "failed to mount on the failover pair $facet,$failover_facet"
+       error_noexit "mount failure on failover pair $facet,$failover_facet"
         return $first_mount_rc
     fi
 
@@ -291,7 +291,7 @@ mount_after_interval() {
     return 0
 }
 
-# Mount the shared target on the failover server 
+# Mount the shared target on the failover server
 # during unmounting it on the primary server.
 mount_during_unmount() {
     local device=$1
@@ -333,7 +333,7 @@ mount_during_unmount() {
     return 0
 }
 
-# Mount the shared target on the failover server 
+# Mount the shared target on the failover server
 # after clean unmounting it on the primary server.
 mount_after_unmount() {
     local device=$1
@@ -347,7 +347,7 @@ mount_after_unmount() {
     start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
 
     log "Unmounting $device on $facet..."
-    stop $facet || return ${PIPESTATUS[0]} 
+    stop $facet || return ${PIPESTATUS[0]}
 
     log "Mounting $device on $failover_facet..."
     start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
@@ -402,6 +402,16 @@ run_e2fsck() {
     return ${PIPESTATUS[0]}
 }
 
+# Run delayed e2fsck on the Lustre server target.
+run_delay_e2fsck() {
+       local facet=$1
+       shift
+       local device=$1
+
+       do_facet $facet "$LUSTRE/tests/e2fsck.exp $device"
+       return ${PIPESTATUS[0]}
+}
+
 # Check whether there are failover pairs for MDS and OSS servers.
 check_failover_pair() {
     [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
@@ -527,35 +537,39 @@ run_test 7 "mount after reboot"
 
 # Test 8 - mount during e2fsck (should never succeed).
 test_8() {
-    local e2fsck_pid
-
-    run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
-    e2fsck_pid=$!
-    sleep 1
+       local e2fsck_pid
+
+       log "Force e2fsck checking on device $MMP_MDSDEV on $MMP_MDS"
+       do_facet $MMP_MDS "$DEBUGFS -w -R 'ssv free_blocks_count 0' $MMP_MDSDEV"
+       run_delay_e2fsck $MMP_MDS $MMP_MDSDEV &
+       e2fsck_pid=$!
+       sleep 5
+
+       if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
+               error_noexit \
+                       "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
+               stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
+               return 1
+       fi
 
-    log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
-    if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
-        error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
-        stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
-        return 1
-    fi
+       wait $e2fsck_pid
 
-    wait $e2fsck_pid
+       echo
+       log "Force e2fsck checking on device $MMP_OSTDEV on $MMP_OSS"
+       do_facet $MMP_OSS "$DEBUGFS -w -R 'ssv free_blocks_count 0' $MMP_OSTDEV"
+       run_delay_e2fsck $MMP_OSS $MMP_OSTDEV &
+       e2fsck_pid=$!
+       sleep 5
 
-    echo
-    run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
-    e2fsck_pid=$!
-    sleep 1
-
-    log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
-    if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
-        error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
-        stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
-        return 2
-    fi
+       if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
+               error_noexit \
+                       "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
+               stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
+               return 2
+       fi
 
-    wait $e2fsck_pid
-    return 0
+       wait $e2fsck_pid
+       return 0
 }
 run_test 8 "mount during e2fsck"
 
@@ -570,7 +584,7 @@ test_9() {
     stop_services primary || return ${PIPESTATUS[0]}
 
     mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
-    
+
     log "Mounting $MMP_MDSDEV on $MMP_MDS..."
     if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
         error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"