LU-1187 tests: Fixes in test-framework for DNE

1. Add dev parameter to mkfs_opts, so it can check if it needs
--mgs for dev. combined_mgs_mdt is only useful for mdt0.
2. Add multi-node facet failover support, so it can fail multiple
targets at the same time.
3. Remove mdsn_HOST/failover_HOSTs in local.sh and add them to
facet_host.
4. replace MDSDEV1 and MGSDEV with mdsdevname and mgsdevname, because
these vars are not always defined.

Signed-off-by: wang di <di.wang@whamcloud.com>
Change-Id: I979b72375a731cf163c6811be3e705ce2298b52c
Reviewed-on: http://review.whamcloud.com/4520
Reviewed-by: Yu Jian <yujian@whamcloud.com>
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index cff13a7..7569790 100644 (file)
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -3,9 +3,7 @@ FSNAME=${FSNAME:-lustre}
 # facet hosts
 mds_HOST=${mds_HOST:-`hostname`}
 mdsfailover_HOST=${mdsfailover_HOST}
-mds1_HOST=${mds1_HOST:-$mds_HOST}
-mds1failover_HOST=${mds1failover_HOST:-$mdsfailover_HOST}
-mgs_HOST=${mgs_HOST:-$mds1_HOST}
+mgs_HOST=${mgs_HOST:-$mds_HOST}
 ost_HOST=${ost_HOST:-`hostname`}
 ostfailover_HOST=${ostfailover_HOST}
 CLIENTS=""
@@ -14,11 +12,6 @@ TMP=${TMP:-/tmp}
 
 DAEMONSIZE=${DAEMONSIZE:-500}
 MDSCOUNT=${MDSCOUNT:-1}
-[ $MDSCOUNT -gt 4 ] && MDSCOUNT=4
-for num in $(seq $MDSCOUNT); do
-    eval mds${num}_HOST=\$\{mds${num}_HOST:-$mds_HOST\}
-    eval mds${num}failover_HOST=\$\{mds${num}failover_HOST:-$mdsfailover_HOST\}
-done
 MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
 MDSSIZE=${MDSSIZE:-200000}
 #
@@ -39,7 +32,6 @@ MDSOPT=${MDSOPT:-}
 MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
 MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
 
-MGSDEV=${MGSDEV:-$MDSDEV1}
 MGSSIZE=${MGSSIZE:-$MDSSIZE}
 MGSOPT=${MGSOPT:-}
 MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index b55c9d3..723fa90 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -901,11 +901,12 @@ test_24a() {
 
        # test 8-char fsname as well
        local FSNAME2=test1234
-       add fs2mds $(mkfs_opts mds1) --nomgs --mgsnode=$MGSNID \
+
+       add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --nomgs --mgsnode=$MGSNID \
                --fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10
 
-       add fs2ost $(mkfs_opts ost1) --fsname=${FSNAME2} --reformat \
-               $fs2ostdev $fs2ostvdev || exit 10
+       add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --fsname=${FSNAME2} \
+               --reformat $fs2ostdev $fs2ostvdev || exit 10
 
        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
@@ -951,8 +952,8 @@ test_24b() {
        local fs2mdsdev=$(mdsdevname 1_2)
        local fs2mdsvdev=$(mdsvdevname 1_2)
 
-       add fs2mds $(mkfs_opts mds1) --mgs --fsname=${FSNAME}2 --reformat \
-               $fs2mdsdev $fs2mdsvdev || exit 10
+       add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --mgs --fsname=${FSNAME}2 \
+               --reformat $fs2mdsdev $fs2mdsvdev || exit 10
        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
        cleanup || return 6
@@ -1638,10 +1639,11 @@ test_33a() { # bug 12333, was test_33
                mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
        fi
 
-       add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
-               $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
-       add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-               --index=8191 --reformat $fs2ostdev $fs2ostvdev || exit 10
+       add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+               --reformat $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
+       add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+               --fsname=${FSNAME2} --index=8191 --reformat $fs2ostdev \
+               $fs2ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -1876,14 +1878,14 @@ test_36() { # 12743
        local fs2ostvdev=$(ostvdevname 1_2)
        local fs3ostvdev=$(ostvdevname 2_2)
 
-       add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
-               $fs2mdsdev $fs2mdsvdev || exit 10
+       add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+               --reformat $fs2mdsdev $fs2mdsvdev || exit 10
        # XXX after we support non 4K disk blocksize in ldiskfs, specify a
        #     different one than the default value here.
-       add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-               --reformat $fs2ostdev $fs2ostvdev || exit 10
-       add fs3ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-               --reformat $fs3ostdev $fs3ostvdev || exit 10
+       add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+               --fsname=${FSNAME2} --reformat $fs2ostdev $fs2ostvdev || exit 10
+       add fs3ost $(mkfs_opts ost1 ${fs3ostdev}) --mgsnode=$MGSNID \
+               --fsname=${FSNAME2} --reformat $fs3ostdev $fs3ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -2976,10 +2978,10 @@ test_55() {
 
        for i in 1023 2048
        do
-               add mds1 $(mkfs_opts mds1) --reformat $mdsdev $mdsvdev ||
-                       exit 10
-               add ost1 $(mkfs_opts ost1) --index=$i --reformat \
-                       $(ostdevname 1) $(ostvdevname 1)
+               add mds1 $(mkfs_opts mds1 ${mdsdev}) --reformat $mdsdev \
+                       $mdsvdev || exit 10
+               add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=$i \
+                       --reformat $(ostdevname 1) $(ostvdevname 1)
                setup_noconfig
                stopall
                setup_noconfig
@@ -3003,10 +3005,14 @@ test_56() {
        local mds_journal_size_orig=$MDSJOURNALSIZE
 
        MDSJOURNALSIZE=16
-       add mds1 $(mkfs_opts mds1) --reformat $(mdsdevname 1) $(mdsvdevname 1)
-       add ost1 $(mkfs_opts ost1) --index=1000 --reformat \
+
+       for num in $(seq 1 $MDSCOUNT); do
+               add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+                       --reformat $(mdsdevname $num) $(mdsvdevname $num)
+       done
+       add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=1000 --reformat \
                $(ostdevname 1) $(ostvdevname 1)
-       add ost2 $(mkfs_opts ost2) --index=10000 --reformat \
+       add ost2 $(mkfs_opts ost2 $(ostdevname 2)) --index=10000 --reformat \
                $(ostdevname 2) $(ostvdevname 2)
 
        start_mgsmds
@@ -3103,14 +3109,19 @@ test_59() {
 run_test 59 "writeconf mount option"
 
 test_60() { # LU-471
+       local num
+
        if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
                skip "Only applicable to ldiskfs-based MDTs"
                return
        fi
 
-       add mds1 $(mkfs_opts mds1) \
-               --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat \
-               $(mdsdevname 1) $(mdsvdevname 1) || exit 10
+       for num in $(seq $MDSCOUNT); do
+               add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+                       --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' \
+                       --reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+                       exit 10
+       done
 
        dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
        rc=${PIPESTATUS[0]}
@@ -3135,10 +3146,13 @@ test_61() { # LU-80
        if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
           ! large_xattr_enabled; then
                reformat=true
-               local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
                LDISKFS_MKFS_OPTS+=" -O large_xattr"
-               add $SINGLEMDS $(mkfs_opts $SINGLEMDS) --reformat $mds_dev ||
-                       error "reformatting $mds_dev failed"
+
+               for num in $(seq $MDSCOUNT); do
+                       add mds${num} $(mkfs_opts mds$num $(mdsdevname $num)) \
+                       --reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+                       error "add mds $num failed"
+               done
        fi
 
     setup_noconfig || error "setting up the filesystem failed"

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 521a58a..921063e 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2045,32 +2045,58 @@ affected_facets () {
 }
 
 facet_failover() {
-    local facet=$1
-    local sleep_time=$2
-    local host=$(facet_active_host $facet)
-
-    echo "Failing $facet on node $host"
-
-    local affected=$(affected_facets $facet)
+       local facets=$1
+       local sleep_time=$2
+       local -a affecteds
+       local facet
+       local total=0
+       local index=0
+       local skip
+
+       #Because it will only get up facets, we need get affected
+       #facets before shutdown
+       #For HARD Failure mode, it needs make sure facets on the same
+       #HOST will only be shutdown and reboot once
+       for facet in ${facets//,/ }; do
+               local affected_facet
+               skip=0
+               #check whether facet has been included in other affected facets
+               for ((index=0; index<$total; index++)); do
+                       [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+               done
 
-    shutdown_facet $facet
+               if [ $skip -eq 0 ]; then
+                       affecteds[$total]=$(affected_facets $facet)
+                       total=$((total+1))
+               fi
+       done
 
-    echo affected facets: $affected
+       for ((index=0; index<$total; index++)); do
+               facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+               local host=$(facet_active_host $facet)
+               echo "Failing ${affecteds[index]} on $host"
+               shutdown_facet $facet
+       done
 
-    [ -n "$sleep_time" ] && sleep $sleep_time
+       for ((index=0; index<$total; index++)); do
+               facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+               echo reboot facets: ${affecteds[index]}
 
-    reboot_facet $facet
+               reboot_facet $facet
 
-    change_active $affected
+               change_active ${affecteds[index]}
 
-    wait_for_facet $affected
-    # start mgs first if it is affected
-    if ! combined_mgs_mds && list_member $affected mgs; then
-        mount_facet mgs || error "Restart of mgs failed"
-    fi
-    # FIXME; has to be changed to mount all facets concurrently
-    affected=$(exclude_items_from_list $affected mgs)
-    mount_facets $affected
+               wait_for_facet ${affecteds[index]}
+               # start mgs first if it is affected
+               if ! combined_mgs_mds &&
+                       list_member ${affecteds[index]} mgs; then
+                       mount_facet mgs || error "Restart of mgs failed"
+               fi
+               # FIXME; has to be changed to mount all facets concurrently
+               affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+               echo mount facets: ${affecteds[index]}
+               mount_facets ${affecteds[index]}
+       done
 }
 
 obd_name() {
@@ -2294,21 +2320,26 @@ hostlist_expand() {
 }
 
 facet_host() {
-    local facet=$1
+       local facet=$1
+       local varname
 
-    [ "$facet" == client ] && echo -n $HOSTNAME && return
-    varname=${facet}_HOST
-    if [ -z "${!varname}" ]; then
-        if [ "${facet:0:3}" == "ost" ]; then
-            eval ${facet}_HOST=${ost_HOST}
-        fi
-    fi
-    echo -n ${!varname}
+       [ "$facet" == client ] && echo -n $HOSTNAME && return
+       varname=${facet}_HOST
+       if [ -z "${!varname}" ]; then
+               if [ "${facet:0:3}" == "ost" ]; then
+                       eval export ${facet}_HOST=${ost_HOST}
+               elif [ "${facet:0:3}" == "mdt" -o \
+                       "${facet:0:3}" == "mds" -o \
+                       "${facet:0:3}" == "mgs" ]; then
+                       eval export ${facet}_HOST=${mds_HOST}
+               fi
+       fi
+       echo -n ${!varname}
 }
 
 facet_failover_host() {
        local facet=$1
-       local var
+       local varname
 
        var=${facet}failover_HOST
        if [ -n "${!var}" ]; then
@@ -2316,12 +2347,18 @@ facet_failover_host() {
                return
        fi
 
+       if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+            "${facet:0:3}" == "mgs" ]; then
+
+               eval export ${facet}failover_host=${mds_HOST}
+               echo ${mds_HOST}
+               return
+       fi
+
        if [[ $facet == ost* ]]; then
-               var=ostfailover_HOST
-               if [ -n "${!var}" ]; then
-                       echo ${!var}
-                       return
-               fi
+               eval export ${facet}failover_host=${ost_HOST}
+               echo ${ost_HOST}
+               return
        fi
 }
 
@@ -2628,9 +2665,10 @@ mdsvdevname() {
 }
 
 mgsdevname() {
-       DEVNAME=MGSDEV
+       local DEVNAME=MGSDEV
+       local MDSDEV1=$(mdsdevname 1)
 
-       local fstype=$(facet_fstype mds$num)
+       local fstype=$(facet_fstype mds1)
 
        case $fstype in
                ldiskfs )
@@ -2649,7 +2687,7 @@ mgsdevname() {
 mgsvdevname() {
        DEVNAME=MGSDEV
 
-       local fstype=$(facet_fstype mds$num)
+       local fstype=$(facet_fstype mds1)
 
        case $fstype in
                ldiskfs )
@@ -2734,7 +2772,8 @@ cleanupall() {
 }
 
 combined_mgs_mds () {
-    [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+       [[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+               [[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
 }
 
 lower() {
@@ -2747,9 +2786,11 @@ upper() {
 
 mkfs_opts() {
        local facet=$1
+       local dev=$2
        local type=$(facet_type $facet)
        local index=$(($(facet_number $facet) - 1))
        local fstype=$(facet_fstype $facet)
+       local host=$(facet_host $facet)
        local opts
        local fs_mkfs_opts
        local var
@@ -2758,7 +2799,9 @@ mkfs_opts() {
                return 1
        fi
 
-       if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+       if [ $type == MGS ] || ( [ $type == MDS ] &&
+                                 [ "$dev" == $(mgsdevname) ] &&
+                                [ "$host" == "$(facet_host mgs)" ] ); then
                opts="--mgs"
        else
                opts="--mgsnode=$MGSNID"
@@ -2835,23 +2878,23 @@ formatall() {
        echo Formatting mgs, mds, osts
        if ! combined_mgs_mds ; then
                echo "Format mgs: $(mgsdevname)"
-               add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
-                       $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
-               fi
+               add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+               $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+       fi
 
-               for num in `seq $MDSCOUNT`; do
-                       echo "Format mds$num: $(mdsdevname $num)"
-                       add mds$num $(mkfs_opts mds$num) --reformat \
-                       $(mdsdevname $num) $(mdsvdevname $num) \
-                       ${quiet:+>/dev/null} || exit 10
-               done
+       for num in $(seq $MDSCOUNT); do
+               echo "Format mds$num: $(mdsdevname $num)"
+               add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+               --reformat $(mdsdevname $num) $(mdsvdevname $num) \
+               ${quiet:+>/dev/null} || exit 10
+       done
 
-               for num in `seq $OSTCOUNT`; do
-                       echo "Format ost$num: $(ostdevname $num)"
-                       add ost$num $(mkfs_opts ost$num) --reformat \
-                       $(ostdevname $num) $(ostvdevname ${num}) \
-                       ${quiet:+>/dev/null} || exit 10
-               done
+       for num in $(seq $OSTCOUNT); do
+               echo "Format ost$num: $(ostdevname $num)"
+               add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+               --reformat $(ostdevname $num) $(ostvdevname ${num}) \
+               ${quiet:+>/dev/null} || exit 10
+       done
 }
 
 mount_client() {
@@ -3044,9 +3087,14 @@ init_facet_vars () {
 
        local varname=${facet}failover_HOST
        if [ -z "${!varname}" ]; then
-               eval $varname=$(facet_host $facet)
+               eval export $varname=$(facet_host $facet)
        fi
 
+       varname=${facet}_HOST
+       if [ -z "${!varname}" ]; then
+               eval export $varname=$(facet_host $facet)
+       fi
+
        # ${facet}failover_dev is set in cfg file
        varname=${facet}failover_dev
        if [ -n "${!varname}" ] ; then
@@ -3068,23 +3116,26 @@ init_facet_vars () {
 }
 
 init_facets_vars () {
-    local DEVNAME
+       local DEVNAME
 
-    if ! remote_mds_nodsh; then 
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=`mdsdevname $num`
-            init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
-        done
-    fi
+       if ! remote_mds_nodsh; then
+               for num in $(seq $MDSCOUNT); do
+                       DEVNAME=`mdsdevname $num`
+                       eval export MDSDEV${num}=$DEVNAME
+                       init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+               done
+       fi
 
+       eval export MGSDEV=$(mgsdevname)
        combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
-    remote_ost_nodsh && return
-
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=`ostdevname $num`
-        init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
-    done
+       if ! remote_ost_nodsh; then
+               for num in $(seq $OSTCOUNT); do
+                       DEVNAME=$(ostdevname $num)
+                       eval export OSTDEV${num}=$DEVNAME
+                       init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+               done
+       fi
 }
 
 osc_ensure_active () {