LU-7169 tests: check disk corruption during failover

[fs/lustre-release.git] / lustre / tests / test-framework.sh

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index e499d7f..2547035 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -23,6 +23,11 @@ export LOAD_LLOOP=${LOAD_LLOOP:-false}
 
 #export PDSH="pdsh -S -Rssh -w"
 export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"}
+export UMOUNT=${UMOUNT:-"umount -d"}
+# sles12 umount has a issue with -d option
+[ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && {
+       export UMOUNT="umount"
+}
 
 # function used by scripts run on remote nodes
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
@@ -594,7 +599,7 @@ load_modules_local() {
        # 'mount' doesn't look in $PATH, just sbin
        local mount_lustre=$LUSTRE/utils/mount.lustre
        if [ -f $mount_lustre ]; then
-               local sbin_mount=/sbin/mount.lustre
+               local sbin_mount=$(readlink -f /sbin)/mount.lustre
                if grep -qw "$sbin_mount" /proc/mounts; then
                        cmp -s $mount_lustre $sbin_mount || umount $sbin_mount
                fi
@@ -658,7 +663,7 @@ unload_modules() {
                fi
        fi
 
-       local sbin_mount=/sbin/mount.lustre
+       local sbin_mount=$(readlink -f /sbin)/mount.lustre
        if grep -qe "$sbin_mount " /proc/mounts; then
                umount $sbin_mount || true
                [ -s $sbin_mount ] && ! grep -q "STUB MARK" $sbin_mount ||
@@ -1240,23 +1245,46 @@ mount_facet() {
                                   ${!dev} $mntpt"
                RC=${PIPESTATUS[0]}
        fi
+
        if [ $RC -ne 0 ]; then
                echo "Start of ${!dev} on ${facet} failed ${RC}"
-    else
-        set_default_debug_facet $facet
+               return $RC
+       fi
+
+       set_default_debug_facet $facet
 
        if [[ $facet == mds* ]]; then
                do_facet $facet \
-                       lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \
-                               2>/dev/null
+               lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null
        fi
 
-               label=$(devicelabel ${facet} ${!dev})
-        [ -z "$label" ] && echo no label for ${!dev} && exit 1
-        eval export ${facet}_svc=${label}
-        echo Started ${label}
-    fi
-    return $RC
+       if [[ $opts =~ .*nosvc.* ]]; then
+               echo "Start ${!dev} without service"
+       else
+               local fstype=$(facet_fstype $facet)
+
+               case $fstype in
+               ldiskfs)
+                       wait_update_facet ${facet} "$E2LABEL ${!dev} \
+                               2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
+                               "" || error "${!dev} failed to initialize!";;
+               zfs)
+                       wait_update_facet ${facet} "$ZFS get -H -o value \
+                               lustre:svname ${!dev} 2>/dev/null | \
+                               grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
+                               error "${!dev} failed to initialize!";;
+
+               *)
+                       error "unknown fstype!";;
+               esac
+       fi
+
+       label=$(devicelabel ${facet} ${!dev})
+       [ -z "$label" ] && echo no label for ${!dev} && exit 1
+       eval export ${facet}_svc=${label}
+       echo Started ${label}
+
+       return $RC
 }
 
 # start facet device options
@@ -1301,7 +1329,7 @@ stop() {
     running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true
     if [ ${running} -ne 0 ]; then
         echo "Stopping $mntpt (opts:$@) on $HOST"
-        do_facet ${facet} umount -d $@ $mntpt
+       do_facet ${facet} $UMOUNT $@ $mntpt
     fi
 
        # umount should block, but we should wait for unrelated obd's
@@ -2432,6 +2460,13 @@ affected_facets () {
 }
 
 facet_failover() {
+       local E2FSCK_ON_MDT0=false
+       if [ "$1" == "--fsck" ]; then
+               shift
+               [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+                       E2FSCK_ON_MDT0=true
+       fi
+
        local facets=$1
        local sleep_time=$2
        local -a affecteds
@@ -2465,6 +2500,9 @@ facet_failover() {
                shutdown_facet $facet
        done
 
+       $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
+               $(mdsdevname 1) "-n" || error "Running e2fsck")
+
        for ((index=0; index<$total; index++)); do
                facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
                echo reboot facets: ${affecteds[index]}
@@ -3222,7 +3260,7 @@ unmount_ldiskfs() {
        local dev=$(facet_device $facet)
        local mnt=$(facet_mntpt $facet)
 
-       do_facet $facet umount -d $mnt
+       do_facet $facet $UMOUNT $mnt
 }
 
 var_name() {
@@ -7006,7 +7044,7 @@ mds_backup_restore() {
        echo "backup data"
        ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3
        # step 6: umount
-       ${rcmd} umount -d $mntpt || return 4
+       ${rcmd} $UMOUNT $mntpt || return 4
        # step 8: reformat dev
        echo "reformat new device"
        format_mdt $(facet_number $facet)
@@ -7022,7 +7060,7 @@ mds_backup_restore() {
        echo "remove recovery logs"
        ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS
        # step 13: umount dev
-       ${rcmd} umount -d $mntpt || return 10
+       ${rcmd} $UMOUNT $mntpt || return 10
        # step 14: cleanup tmp backup
        ${rcmd} rm -f $metaea $metadata
        # step 15: reset device label - it's not virgin on
@@ -7062,7 +7100,7 @@ mds_remove_ois() {
                done
        fi
        # step 4: umount
-       ${rcmd} umount -d $mntpt || return 2
+       ${rcmd} $UMOUNT $mntpt || return 2
        # OI files will be recreated when mounted as lustre next time.
 }