LU-1689 tests: fix mount during e2fsck test

The current mmp test 8 (mount during e2fsck) has two time issues:
1) the mount operation may start before e2fsck
2) the e2fsck operation may stop before mount

This patch fixes the above issues by providing enough time for e2fsck
operation to be started before mount operation, and setting the
superblock free_blocks_count field with 0 to force e2fsck checking
the Lustre server target device, which provides enough time for
the mount operation to be started during the e2fsck operation.

Test-Parameters: testlist=mmp
Signed-off-by: Yu Jian <yujian@whamcloud.com>
Change-Id: I5be2b84f063a0db386a8d9d48db53c00ebd77864
Reviewed-on: http://review.whamcloud.com/3643
Reviewed-by: Li Wei <liwei@whamcloud.com>
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index a7f32c5..da2e6ea 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -27,7 +27,7 @@ noinst_SCRIPTS += lustre-rsync-test.sh ost-pools.sh rpc.sh yaml.sh liblustre.sh
 noinst_SCRIPTS += lnet-selftest.sh obdfilter-survey.sh mmp.sh mmp_mark.sh
 noinst_SCRIPTS += sgpdd-survey.sh maloo_upload.sh auster setup-nfs.sh
 noinst_SCRIPTS += parallel-scale-nfsv3.sh parallel-scale-nfsv4.sh
-noinst_SCRIPTS += parallel-scale-nfs.sh
+noinst_SCRIPTS += parallel-scale-nfs.sh e2fsck.exp
 nobase_noinst_SCRIPTS = cfg/local.sh
 nobase_noinst_SCRIPTS += test-groups/regression test-groups/regression-mpi
 nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh

diff --git a/lustre/tests/e2fsck.exp b/lustre/tests/e2fsck.exp
new file mode 100755 (executable)
index 0000000..12bdce7
--- /dev/null
+++ b/lustre/tests/e2fsck.exp
@@ -0,0 +1,28 @@
+#!/usr/bin/expect -f
+#
+# This expect script invokes e2fsck to check the filesystem
+# on a device and allows e2fsck to be run interactively.
+#
+set env(PATH) "/usr/bin:/bin:/usr/sbin:/sbin"
+
+log_user 1
+
+if {$argc == 1} {
+    set dev [lindex $argv 0]
+} else {
+    set prog [lindex [split $argv0 "/"] end]
+    send_user "Usage: $prog <device>\r\n"
+    exit 1
+}
+
+spawn e2fsck -f $dev
+expect {
+    -gl "<y>? " {
+        sleep 30
+        send "yes\r"
+        exp_continue
+    }
+    -gl "FILE SYSTEM WAS MODIFIED" {
+        exit 0
+    }
+}

diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh
index cb35a27..8195711 100755 (executable)
--- a/lustre/tests/mmp.sh
+++ b/lustre/tests/mmp.sh
@@ -215,7 +215,7 @@ mmp_fini() {
     return 0
 }
 
-# Mount the shared target on the failover server after some interval it's 
+# Mount the shared target on the failover server after some interval it's
 # mounted on the primary server.
 mount_after_interval_sub() {
     local interval=$1
@@ -254,7 +254,7 @@ mount_after_interval_sub() {
             return ${PIPESTATUS[0]}
         return 1
     elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
-        error_noexit "failed to mount on the failover pair $facet,$failover_facet"
+        error_noexit "mount failure on failover pair $facet,$failover_facet"
         return $first_mount_rc
     fi
 
@@ -280,7 +280,7 @@ mount_after_interval() {
     return 0
 }
 
-# Mount the shared target on the failover server 
+# Mount the shared target on the failover server
 # during unmounting it on the primary server.
 mount_during_unmount() {
     local device=$1
@@ -322,7 +322,7 @@ mount_during_unmount() {
     return 0
 }
 
-# Mount the shared target on the failover server 
+# Mount the shared target on the failover server
 # after clean unmounting it on the primary server.
 mount_after_unmount() {
     local device=$1
@@ -336,7 +336,7 @@ mount_after_unmount() {
     start $facet $device $mnt_opts || return ${PIPESTATUS[0]}
 
     log "Unmounting $device on $facet..."
-    stop $facet || return ${PIPESTATUS[0]} 
+    stop $facet || return ${PIPESTATUS[0]}
 
     log "Mounting $device on $failover_facet..."
     start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}
@@ -391,6 +391,16 @@ run_e2fsck() {
     return ${PIPESTATUS[0]}
 }
 
+# Run delayed e2fsck on the Lustre server target.
+run_delay_e2fsck() {
+    local facet=$1
+    shift
+    local device=$1
+
+    do_facet $facet "$LUSTRE/tests/e2fsck.exp $device"
+    return ${PIPESTATUS[0]}
+}
+
 # Check whether there are failover pairs for MDS and OSS servers.
 check_failover_pair() {
     [ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] \
@@ -518,13 +528,15 @@ run_test 7 "mount after reboot"
 test_8() {
     local e2fsck_pid
 
-    run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
+    log "Force e2fsck checking on device $MMP_MDSDEV on $MMP_MDS"
+    do_facet $MMP_MDS "$DEBUGFS -w -R 'ssv free_blocks_count 0' $MMP_MDSDEV"
+    run_delay_e2fsck $MMP_MDS $MMP_MDSDEV &
     e2fsck_pid=$!
-    sleep 1
+    sleep 5
 
-    log "Mounting $MMP_MDSDEV on $MMP_MDS_FAILOVER..."
     if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
-        error_noexit "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
+        error_noexit \
+            "mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
         stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
         return 1
     fi
@@ -532,13 +544,15 @@ test_8() {
     wait $e2fsck_pid
 
     echo
-    run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
+    log "Force e2fsck checking on device $MMP_OSTDEV on $MMP_OSS"
+    do_facet $MMP_OSS "$DEBUGFS -w -R 'ssv free_blocks_count 0' $MMP_OSTDEV"
+    run_delay_e2fsck $MMP_OSS $MMP_OSTDEV &
     e2fsck_pid=$!
-    sleep 1
+    sleep 5
 
-    log "Mounting $MMP_OSTDEV on $MMP_OSS_FAILOVER..."
     if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
-        error_noexit "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
+        error_noexit \
+            "mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
         stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
         return 2
     fi
@@ -559,7 +573,7 @@ test_9() {
     stop_services primary || return ${PIPESTATUS[0]}
 
     mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}
-    
+
     log "Mounting $MMP_MDSDEV on $MMP_MDS..."
     if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
         error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"