From: wangdi <di.wang@whamcloud.com>
Date: Mon, 12 Nov 2012 22:17:00 +0000 (-0800)
Subject: LU-1187 tests: Fixes in test-framework for DNE
X-Git-Tag: 2.3.57~16
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=682f3e19473b91f6face2567513b25a7b6d3fc24

LU-1187 tests: Fixes in test-framework for DNE

1. Add dev parameter to mkfs_opts, so it can check if it needs
--mgs for dev. combined_mgs_mdt is only useful for mdt0.
2. Add multi-node facet failover support, so it can fail multiple
targets at the same time.
3. Remove mdsn_HOST/failover_HOSTs in local.sh and add them to
facet_host.
4. replace MDSDEV1 and MGSDEV with mdsdevname and mgsdevname, because
these vars are not always defined.

Signed-off-by: wang di <di.wang@whamcloud.com>
Change-Id: I979b72375a731cf163c6811be3e705ce2298b52c
Reviewed-on: http://review.whamcloud.com/4520
Reviewed-by: Yu Jian <yujian@whamcloud.com>
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index cff13a7..7569790 100644
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -3,9 +3,7 @@ FSNAME=${FSNAME:-lustre}
 # facet hosts
 mds_HOST=${mds_HOST:-`hostname`}
 mdsfailover_HOST=${mdsfailover_HOST}
-mds1_HOST=${mds1_HOST:-$mds_HOST}
-mds1failover_HOST=${mds1failover_HOST:-$mdsfailover_HOST}
-mgs_HOST=${mgs_HOST:-$mds1_HOST}
+mgs_HOST=${mgs_HOST:-$mds_HOST}
 ost_HOST=${ost_HOST:-`hostname`}
 ostfailover_HOST=${ostfailover_HOST}
 CLIENTS=""
@@ -14,11 +12,6 @@ TMP=${TMP:-/tmp}
 
 DAEMONSIZE=${DAEMONSIZE:-500}
 MDSCOUNT=${MDSCOUNT:-1}
-[ $MDSCOUNT -gt 4 ] && MDSCOUNT=4
-for num in $(seq $MDSCOUNT); do
-    eval mds${num}_HOST=\$\{mds${num}_HOST:-$mds_HOST\}
-    eval mds${num}failover_HOST=\$\{mds${num}failover_HOST:-$mdsfailover_HOST\}
-done
 MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
 MDSSIZE=${MDSSIZE:-200000}
 #
@@ -39,7 +32,6 @@ MDSOPT=${MDSOPT:-}
 MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
 MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
 
-MGSDEV=${MGSDEV:-$MDSDEV1}
 MGSSIZE=${MGSSIZE:-$MDSSIZE}
 MGSOPT=${MGSOPT:-}
 MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index b55c9d3..723fa90 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -901,11 +901,12 @@ test_24a() {
 
 	# test 8-char fsname as well
 	local FSNAME2=test1234
-	add fs2mds $(mkfs_opts mds1) --nomgs --mgsnode=$MGSNID \
+
+	add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --nomgs --mgsnode=$MGSNID \
 		--fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10
 
-	add fs2ost $(mkfs_opts ost1) --fsname=${FSNAME2} --reformat \
-		$fs2ostdev $fs2ostvdev || exit 10
+	add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --fsname=${FSNAME2} \
+		--reformat $fs2ostdev $fs2ostvdev || exit 10
 
 	setup
 	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
@@ -951,8 +952,8 @@ test_24b() {
 	local fs2mdsdev=$(mdsdevname 1_2)
 	local fs2mdsvdev=$(mdsvdevname 1_2)
 
-	add fs2mds $(mkfs_opts mds1) --mgs --fsname=${FSNAME}2 --reformat \
-		$fs2mdsdev $fs2mdsvdev || exit 10
+	add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --mgs --fsname=${FSNAME}2 \
+		--reformat $fs2mdsdev $fs2mdsvdev || exit 10
 	setup
 	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
 	cleanup || return 6
@@ -1638,10 +1639,11 @@ test_33a() { # bug 12333, was test_33
 		mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
 	fi
 
-	add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
-		$mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
-	add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-		--index=8191 --reformat $fs2ostdev $fs2ostvdev || exit 10
+	add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+		--reformat $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
+	add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+		--fsname=${FSNAME2} --index=8191 --reformat $fs2ostdev \
+		$fs2ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -1876,14 +1878,14 @@ test_36() { # 12743
 	local fs2ostvdev=$(ostvdevname 1_2)
 	local fs3ostvdev=$(ostvdevname 2_2)
 
-	add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
-		$fs2mdsdev $fs2mdsvdev || exit 10
+	add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+		--reformat $fs2mdsdev $fs2mdsvdev || exit 10
 	# XXX after we support non 4K disk blocksize in ldiskfs, specify a
 	#     different one than the default value here.
-	add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-		--reformat $fs2ostdev $fs2ostvdev || exit 10
-	add fs3ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
-		--reformat $fs3ostdev $fs3ostvdev || exit 10
+	add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+		--fsname=${FSNAME2} --reformat $fs2ostdev $fs2ostvdev || exit 10
+	add fs3ost $(mkfs_opts ost1 ${fs3ostdev}) --mgsnode=$MGSNID \
+		--fsname=${FSNAME2} --reformat $fs3ostdev $fs3ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -2976,10 +2978,10 @@ test_55() {
 
 	for i in 1023 2048
 	do
-		add mds1 $(mkfs_opts mds1) --reformat $mdsdev $mdsvdev ||
-			exit 10
-		add ost1 $(mkfs_opts ost1) --index=$i --reformat \
-			$(ostdevname 1) $(ostvdevname 1)
+		add mds1 $(mkfs_opts mds1 ${mdsdev}) --reformat $mdsdev \
+			$mdsvdev || exit 10
+		add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=$i \
+			--reformat $(ostdevname 1) $(ostvdevname 1)
 		setup_noconfig
 		stopall
 		setup_noconfig
@@ -3003,10 +3005,14 @@ test_56() {
 	local mds_journal_size_orig=$MDSJOURNALSIZE
 
 	MDSJOURNALSIZE=16
-	add mds1 $(mkfs_opts mds1) --reformat $(mdsdevname 1) $(mdsvdevname 1)
-	add ost1 $(mkfs_opts ost1) --index=1000 --reformat \
+
+	for num in $(seq 1 $MDSCOUNT); do
+		add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+			--reformat $(mdsdevname $num) $(mdsvdevname $num)
+	done
+	add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=1000 --reformat \
 		$(ostdevname 1) $(ostvdevname 1)
-	add ost2 $(mkfs_opts ost2) --index=10000 --reformat \
+	add ost2 $(mkfs_opts ost2 $(ostdevname 2)) --index=10000 --reformat \
 		$(ostdevname 2) $(ostvdevname 2)
 
 	start_mgsmds
@@ -3103,14 +3109,19 @@ test_59() {
 run_test 59 "writeconf mount option"
 
 test_60() { # LU-471
+	local num
+
 	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
 		skip "Only applicable to ldiskfs-based MDTs"
 		return
 	fi
 
-	add mds1 $(mkfs_opts mds1) \
-		--mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat \
-		$(mdsdevname 1) $(mdsvdevname 1) || exit 10
+	for num in $(seq $MDSCOUNT); do
+		add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+			--mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' \
+			--reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+			exit 10
+	done
 
 	dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
 	rc=${PIPESTATUS[0]}
@@ -3135,10 +3146,13 @@ test_61() { # LU-80
 	if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
 	   ! large_xattr_enabled; then
 		reformat=true
-		local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
 		LDISKFS_MKFS_OPTS+=" -O large_xattr"
-		add $SINGLEMDS $(mkfs_opts $SINGLEMDS) --reformat $mds_dev ||
-			error "reformatting $mds_dev failed"
+
+		for num in $(seq $MDSCOUNT); do
+			add mds${num} $(mkfs_opts mds$num $(mdsdevname $num)) \
+			--reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+			error "add mds $num failed"
+		done
 	fi
 
     setup_noconfig || error "setting up the filesystem failed"
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 521a58a..921063e 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2045,32 +2045,58 @@ affected_facets () {
 }
 
 facet_failover() {
-    local facet=$1
-    local sleep_time=$2
-    local host=$(facet_active_host $facet)
-
-    echo "Failing $facet on node $host"
-
-    local affected=$(affected_facets $facet)
+	local facets=$1
+	local sleep_time=$2
+	local -a affecteds
+	local facet
+	local total=0
+	local index=0
+	local skip
+
+	#Because it will only get up facets, we need get affected
+	#facets before shutdown
+	#For HARD Failure mode, it needs make sure facets on the same
+	#HOST will only be shutdown and reboot once
+	for facet in ${facets//,/ }; do
+		local affected_facet
+		skip=0
+		#check whether facet has been included in other affected facets
+		for ((index=0; index<$total; index++)); do
+			[[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+		done
 
-    shutdown_facet $facet
+		if [ $skip -eq 0 ]; then
+			affecteds[$total]=$(affected_facets $facet)
+			total=$((total+1))
+		fi
+	done
 
-    echo affected facets: $affected
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		local host=$(facet_active_host $facet)
+		echo "Failing ${affecteds[index]} on $host"
+		shutdown_facet $facet
+	done
 
-    [ -n "$sleep_time" ] && sleep $sleep_time
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		echo reboot facets: ${affecteds[index]}
 
-    reboot_facet $facet
+		reboot_facet $facet
 
-    change_active $affected
+		change_active ${affecteds[index]}
 
-    wait_for_facet $affected
-    # start mgs first if it is affected
-    if ! combined_mgs_mds && list_member $affected mgs; then
-        mount_facet mgs || error "Restart of mgs failed"
-    fi
-    # FIXME; has to be changed to mount all facets concurrently
-    affected=$(exclude_items_from_list $affected mgs)
-    mount_facets $affected
+		wait_for_facet ${affecteds[index]}
+		# start mgs first if it is affected
+		if ! combined_mgs_mds &&
+			list_member ${affecteds[index]} mgs; then
+			mount_facet mgs || error "Restart of mgs failed"
+		fi
+		# FIXME; has to be changed to mount all facets concurrently
+		affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+		echo mount facets: ${affecteds[index]}
+		mount_facets ${affecteds[index]}
+	done
 }
 
 obd_name() {
@@ -2294,21 +2320,26 @@ hostlist_expand() {
 }
 
 facet_host() {
-    local facet=$1
+	local facet=$1
+	local varname
 
-    [ "$facet" == client ] && echo -n $HOSTNAME && return
-    varname=${facet}_HOST
-    if [ -z "${!varname}" ]; then
-        if [ "${facet:0:3}" == "ost" ]; then
-            eval ${facet}_HOST=${ost_HOST}
-        fi
-    fi
-    echo -n ${!varname}
+	[ "$facet" == client ] && echo -n $HOSTNAME && return
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		if [ "${facet:0:3}" == "ost" ]; then
+			eval export ${facet}_HOST=${ost_HOST}
+		elif [ "${facet:0:3}" == "mdt" -o \
+			"${facet:0:3}" == "mds" -o \
+			"${facet:0:3}" == "mgs" ]; then
+			eval export ${facet}_HOST=${mds_HOST}
+		fi
+	fi
+	echo -n ${!varname}
 }
 
 facet_failover_host() {
 	local facet=$1
-	local var
+	local varname
 
 	var=${facet}failover_HOST
 	if [ -n "${!var}" ]; then
@@ -2316,12 +2347,18 @@ facet_failover_host() {
 		return
 	fi
 
+	if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+	     "${facet:0:3}" == "mgs" ]; then
+
+		eval export ${facet}failover_host=${mds_HOST}
+		echo ${mds_HOST}
+		return
+	fi
+
 	if [[ $facet == ost* ]]; then
-		var=ostfailover_HOST
-		if [ -n "${!var}" ]; then
-			echo ${!var}
-			return
-		fi
+		eval export ${facet}failover_host=${ost_HOST}
+		echo ${ost_HOST}
+		return
 	fi
 }
 
@@ -2628,9 +2665,10 @@ mdsvdevname() {
 }
 
 mgsdevname() {
-	DEVNAME=MGSDEV
+	local DEVNAME=MGSDEV
+	local MDSDEV1=$(mdsdevname 1)
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
 		ldiskfs )
@@ -2649,7 +2687,7 @@ mgsdevname() {
 mgsvdevname() {
 	DEVNAME=MGSDEV
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
 		ldiskfs )
@@ -2734,7 +2772,8 @@ cleanupall() {
 }
 
 combined_mgs_mds () {
-    [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+	[[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+		[[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
 }
 
 lower() {
@@ -2747,9 +2786,11 @@ upper() {
 
 mkfs_opts() {
 	local facet=$1
+	local dev=$2
 	local type=$(facet_type $facet)
 	local index=$(($(facet_number $facet) - 1))
 	local fstype=$(facet_fstype $facet)
+	local host=$(facet_host $facet)
 	local opts
 	local fs_mkfs_opts
 	local var
@@ -2758,7 +2799,9 @@ mkfs_opts() {
 		return 1
 	fi
 
-	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+	if [ $type == MGS ] || ( [ $type == MDS ] &&
+                                 [ "$dev" == $(mgsdevname) ] &&
+				 [ "$host" == "$(facet_host mgs)" ] ); then
 		opts="--mgs"
 	else
 		opts="--mgsnode=$MGSNID"
@@ -2835,23 +2878,23 @@ formatall() {
 	echo Formatting mgs, mds, osts
 	if ! combined_mgs_mds ; then
 		echo "Format mgs: $(mgsdevname)"
-		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
-			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
-		fi
+		add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+		$(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+	fi
 
-		for num in `seq $MDSCOUNT`; do
-			echo "Format mds$num: $(mdsdevname $num)"
-			add mds$num $(mkfs_opts mds$num) --reformat \
-			$(mdsdevname $num) $(mdsvdevname $num) \
-			${quiet:+>/dev/null} || exit 10
-		done
+	for num in $(seq $MDSCOUNT); do
+		echo "Format mds$num: $(mdsdevname $num)"
+		add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+		--reformat $(mdsdevname $num) $(mdsvdevname $num) \
+		${quiet:+>/dev/null} || exit 10
+	done
 
-		for num in `seq $OSTCOUNT`; do
-			echo "Format ost$num: $(ostdevname $num)"
-			add ost$num $(mkfs_opts ost$num) --reformat \
-			$(ostdevname $num) $(ostvdevname ${num}) \
-			${quiet:+>/dev/null} || exit 10
-		done
+	for num in $(seq $OSTCOUNT); do
+		echo "Format ost$num: $(ostdevname $num)"
+		add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+		--reformat $(ostdevname $num) $(ostvdevname ${num}) \
+		${quiet:+>/dev/null} || exit 10
+	done
 }
 
 mount_client() {
@@ -3044,9 +3087,14 @@ init_facet_vars () {
 
 	local varname=${facet}failover_HOST
 	if [ -z "${!varname}" ]; then
-		eval $varname=$(facet_host $facet)
+		eval export $varname=$(facet_host $facet)
 	fi
 
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		eval export $varname=$(facet_host $facet)
+ 	fi
+
 	# ${facet}failover_dev is set in cfg file
 	varname=${facet}failover_dev
 	if [ -n "${!varname}" ] ; then
@@ -3068,23 +3116,26 @@ init_facet_vars () {
 }
 
 init_facets_vars () {
-    local DEVNAME
+	local DEVNAME
 
-    if ! remote_mds_nodsh; then 
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=`mdsdevname $num`
-            init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
-        done
-    fi
+	if ! remote_mds_nodsh; then
+		for num in $(seq $MDSCOUNT); do
+			DEVNAME=`mdsdevname $num`
+			eval export MDSDEV${num}=$DEVNAME
+			init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+		done
+	fi
 
+	eval export MGSDEV=$(mgsdevname)
 	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
-    remote_ost_nodsh && return
-
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=`ostdevname $num`
-        init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
-    done
+	if ! remote_ost_nodsh; then
+		for num in $(seq $OSTCOUNT); do
+			DEVNAME=$(ostdevname $num)
+			eval export OSTDEV${num}=$DEVNAME
+			init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+		done
+	fi
 }
 
 osc_ensure_active () {