X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fconf-sanity.sh;h=d6087f600a6307443d61fea637567d11252c1c14;hp=546a0d89ddc6f07e3861dcc9974b1f368eb9493a;hb=5735341e9d343d452930c8cdc5a00b09c84db93b;hpb=2358e27bc14683129a4c9f69f880c20d2e65db34

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 546a0d8..d6087f6 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -12,23 +12,29 @@ set -e
 ONLY=${ONLY:-"$*"}
 
 # These tests don't apply to mountconf
-#              xml xml xml xml xml xml dumb FIXME
-MOUNTCONFSKIP="10  11  12  13  13b 14  15   18"
+MOUNTCONFSKIP="9 10 11 12 13 13b 14 15"
 
-# bug number for skipped test:
-ALWAYS_EXCEPT=" $CONF_SANITY_EXCEPT $MOUNTCONFSKIP"
+# bug number for skipped test:                     2661 13709 10510 13710 12743
+ALWAYS_EXCEPT=" $CONF_SANITY_EXCEPT $MOUNTCONFSKIP 16   22    23    33    36"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 SRCDIR=`dirname $0`
 PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH
 
+PTLDEBUG=${PTLDEBUG:--1}
+SAVE_PWD=$PWD
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 RLUSTRE=${RLUSTRE:-$LUSTRE}
+MOUNTLUSTRE=${MOUNTLUSTRE:-/sbin/mount.lustre}
+MKFSLUSTRE=${MKFSLUSTRE:-/usr/sbin/mkfs.lustre}
 HOSTNAME=`hostname`
 
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+# use small MDS + OST size to speed formatting time
+MDSSIZE=40000
+OSTSIZE=40000
+. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
 
 reformat() {
         formatall
@@ -163,6 +169,7 @@ fi
 
 gen_config
 
+init_krb5_env
 
 test_0() {
         setup
@@ -292,8 +299,6 @@ run_test 5d "mount with ost down"
 test_5e() {
 	start_ost
 	start_mds
-        # give MDS a chance to connect to OSTs (bz 10476)
-	sleep 5	
 
 #define OBD_FAIL_PTLRPC_DELAY_SEND       0x506
 	do_facet client "sysctl -w lustre.fail_loc=0x80000506"
@@ -596,7 +601,7 @@ test_15() {
 		echo "save $MOUNTLUSTRE to $MOUNTLUSTRE.sav"
 		mv $MOUNTLUSTRE $MOUNTLUSTRE.sav && trap cleanup_15 EXIT INT
 		if [ -f $MOUNTLUSTRE ]; then
-			echo "$MOUNTLUSTRE cannot be moved, skipping test"
+			skip "$MOUNTLUSTRE cannot be moved, skipping test"
 			return 0
 		fi
 	fi
@@ -611,54 +616,53 @@ run_test 15 "zconf-mount without /sbin/mount.lustre (should return error)"
 test_16() {
         TMPMTPT="${MOUNT%/*}/conf16"
 
-        if [ ! -f "$MDSDEV" ]; then
-            echo "no $MDSDEV existing, so mount Lustre to create one"
+        if [ ! -e "$MDSDEV" ]; then
+            log "no $MDSDEV existing, so mount Lustre to create one"
 	    setup
             check_mount || return 41
             cleanup || return $?
         fi
 
-        echo "change the mode of $MDSDEV/OBJECTS,LOGS,PENDING to 555"
+        [ -f "$MDSDEV" ] && LOOPOPT="-o loop"
+
+        log "change the mode of $MDSDEV/OBJECTS,LOGS,PENDING to 555"
         do_facet mds "mkdir -p $TMPMTPT &&
-                      mount -o loop -t ext3 $MDSDEV $TMPMTPT &&
+                      mount $LOOPOPT -t $FSTYPE $MDSDEV $TMPMTPT &&
                       chmod 555 $TMPMTPT/{OBJECTS,LOGS,PENDING} &&
                       umount $TMPMTPT" || return $?
 
-        echo "mount Lustre to change the mode of OBJECTS/LOGS/PENDING, then umount Lustre"
+        log "mount Lustre to change the mode of OBJECTS/LOGS/PENDING, then umount Lustre"
 	setup
         check_mount || return 41
         cleanup || return $?
 
-        echo "read the mode of OBJECTS/LOGS/PENDING and check if they has been changed properly"
+        log "read the mode of OBJECTS/LOGS/PENDING and check if they has been changed properly"
         EXPECTEDOBJECTSMODE=`do_facet mds "debugfs -R 'stat OBJECTS' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
         EXPECTEDLOGSMODE=`do_facet mds "debugfs -R 'stat LOGS' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
         EXPECTEDPENDINGMODE=`do_facet mds "debugfs -R 'stat PENDING' $MDSDEV 2> /dev/null" | grep 'Mode: ' | sed -e "s/.*Mode: *//" -e "s/ *Flags:.*//"`
 
         if [ "$EXPECTEDOBJECTSMODE" = "0777" ]; then
-                echo "Success:Lustre change the mode of OBJECTS correctly"
+                log "Success:Lustre change the mode of OBJECTS correctly"
         else
-                echo "Error: Lustre does not change mode of OBJECTS properly"
-                return 1
+                error "Lustre does not change mode of OBJECTS properly"
         fi
 
         if [ "$EXPECTEDLOGSMODE" = "0777" ]; then
-                echo "Success:Lustre change the mode of LOGS correctly"
+                log "Success:Lustre change the mode of LOGS correctly"
         else
-                echo "Error: Lustre does not change mode of LOGS properly"
-                return 1
+                error "Lustre does not change mode of LOGS properly"
         fi
 
         if [ "$EXPECTEDPENDINGMODE" = "0777" ]; then
-                echo "Success:Lustre change the mode of PENDING correctly"
+                log "Success:Lustre change the mode of PENDING correctly"
         else
-                echo "Error: Lustre does not change mode of PENDING properly"
-                return 1
+                error "Lustre does not change mode of PENDING properly"
         fi
 }
 run_test 16 "verify that lustre will correct the mode of OBJECTS/LOGS/PENDING"
 
 test_17() {
-        if [ ! -f "$MDSDEV" ]; then
+        if [ ! -e "$MDSDEV" ]; then
             echo "no $MDSDEV existing, so mount Lustre to create one"
 	    setup
             check_mount || return 41
@@ -677,31 +681,30 @@ run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should ret
 test_18() {
         [ -f $MDSDEV ] && echo "remove $MDSDEV" && rm -f $MDSDEV
         echo "mount mds with large journal..."
-        OLDMDSSIZE=$MDSSIZE
-        MDSSIZE=2000000
-	#FIXME have to change MDS_MKFS_OPTS
-        gen_config
+        local myMDSSIZE=2000000
+        OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
 
+        MDS_MKFS_OPTS="--mgs --mdt --fsname=$FSNAME --device-size=$myMDSSIZE --param sys.timeout=$TIMEOUT $MDSOPT"
+
+        gen_config
         echo "mount lustre system..."
 	setup
         check_mount || return 41
 
         echo "check journal size..."
-        FOUNDJOURNALSIZE=`do_facet mds "debugfs -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}'`
-        if [ "$FOUNDJOURNALSIZE" = "79691776" ]; then
-                echo "Success:lconf creates large journals"
+        FOUNDSIZE=`do_facet mds "debugfs -c -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}'`
+        if [ $FOUNDSIZE -gt $((32 * 1024 * 1024)) ]; then
+                log "Success: mkfs creates large journals. Size: $((FOUNDSIZE >> 20))M"
         else
-                echo "Error:lconf not create large journals correctly"
-                echo "expected journal size: 79691776(76M), found journal size: $FOUNDJOURNALSIZE"
-                return 1
+                error "expected journal size > 32M, found $((FOUNDSIZE >> 20))M"
         fi
 
         cleanup || return $?
 
-        MDSSIZE=$OLDMDSSIZE
+        MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
         gen_config
 }
-run_test 18 "check lconf creates large journals"
+run_test 18 "check mkfs creates large journals"
 
 test_19a() {
 	start_mds || return 1
@@ -763,8 +766,8 @@ test_21c() {
 run_test 21c "start mds between two osts, stop mds last"
 
 test_22() {
-        #reformat to remove all logs
-        reformat
+	#reformat to remove all logs
+	reformat
 	start_mds
 	echo Client mount before any osts are in the logs
 	mount_client $MOUNT
@@ -782,7 +785,6 @@ test_22() {
 	echo Client mount with a running ost
 	start_ost
 	mount_client $MOUNT
-	sleep 5	#bz10476
 	check_mount || return 41
 	pass
 
@@ -819,17 +821,28 @@ test_23() {
 #run_test 23 "interrupt client during recovery mount delay"
 
 test_24a() {
+	#set up fs1 
+	gen_config
+	#set up fs2
 	local fs2mds_HOST=$mds_HOST
+	local fs2ost_HOST=$ost_HOST
+	[ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
+	if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" ]; then
+		do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
+		skip "mixed loopback and real device not working" && return
+	fi
+
+	local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+	local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
+
 	# test 8-char fsname as well
 	local FSNAME2=test1234
-        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat ${MDSDEV}_2 || exit 10
+	add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat $fs2mdsdev || exit 10
 
-	local fs2ost_HOST=$ost_HOST
-	local fs2ostdev=$(ostdevname 1)_2
 	add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2ostdev || exit 10
 
 	setup
-	start fs2mds ${MDSDEV}_2 $MDS_MOUNT_OPTS
+	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
 	start fs2ost $fs2ostdev $OST_MOUNT_OPTS
 	mkdir -p $MOUNT2
 	mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
@@ -840,6 +853,7 @@ test_24a() {
 	sleep 10
 	[ -e $MOUNT2/$tfile ] && error "File bleed" && return 7
 	# 2 should work
+	sleep 5
 	cp /etc/passwd $MOUNT2/b || return 3
 	rm $MOUNT2/b || return 4
 	# 2 is actually mounted
@@ -851,8 +865,8 @@ test_24a() {
  	umount_client $MOUNT 
 	# the MDS must remain up until last MDT
 	stop_mds
-	MDS=$(awk '($3 ~ "mdt" && $4 ~ "MDS") { print $4 }' $LPROC/devices)
-	[ -z "$MDS" ] && error "No MDS" && return 8
+	MDS=$(do_facet $SINGLEMDS "cat $LPROC/devices" | awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }')
+	[ -z "$MDS" ] && error "No MDT" && return 8
 	umount $MOUNT2
 	stop fs2mds -f
 	stop fs2ost -f
@@ -862,9 +876,16 @@ run_test 24a "Multiple MDTs on a single node"
 
 test_24b() {
 	local fs2mds_HOST=$mds_HOST
-        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat ${MDSDEV}_2 || exit 10
+	if [ -z "$fs2mds_DEV" ]; then
+		do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
+		skip "mixed loopback and real device not working" && return
+	fi
+
+	local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+
+	add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat $fs2mdsdev || exit 10 
 	setup
-	start fs2mds ${MDSDEV}_2 $MDS_MOUNT_OPTS && return 2
+	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
 	cleanup || return 6
 }
 run_test 24b "Multiple MGSs on a single node (should return err)"
@@ -881,9 +902,9 @@ run_test 25 "Verify modules are referenced"
 test_26() {
     load_modules
     # we need modules before mount for sysctl, so make sure...
-    [ -z "$(lsmod | grep lustre)" ] && modprobe lustre 
+    do_facet mds "lsmod | grep -q lustre || modprobe lustre"
 #define OBD_FAIL_MDS_FS_SETUP            0x135
-    sysctl -w lustre.fail_loc=0x80000135
+    do_facet mds "sysctl -w lustre.fail_loc=0x80000135"
     start_mds && echo MDS started && return 1
     cat $LPROC/devices
     DEVS=$(cat $LPROC/devices | wc -l)
@@ -893,23 +914,24 @@ test_26() {
 run_test 26 "MDT startup failure cleans LOV (should return errs)"
 
 set_and_check() {
-        local TEST=$1
-	local PARAM=$2
-	local ORIG=$($TEST) 
-	if [ $# -gt 2 ]; then
-	    local FINAL=$3
+	local myfacet=$1
+	local TEST=$2
+	local PARAM=$3
+	local ORIG=$(do_facet $myfacet "$TEST") 
+	if [ $# -gt 3 ]; then
+	    local FINAL=$4
 	else
 	    local -i FINAL
 	    FINAL=$(($ORIG + 5))
 	fi
 	echo "Setting $PARAM from $ORIG to $FINAL"
-	$LCTL conf_param $PARAM=$FINAL
+	do_facet mds "$LCTL conf_param $PARAM=$FINAL" || error conf_param failed
 	local RESULT
-	local MAX=20
+	local MAX=90
 	local WAIT=0
 	while [ 1 ]; do
 	    sleep 5
-	    RESULT=$($TEST) 
+	    RESULT=$(do_facet $myfacet "$TEST") 
 	    if [ $RESULT -eq $FINAL ]; then
 		echo "Updated config after $WAIT sec (got $RESULT)"
 		break
@@ -928,7 +950,7 @@ test_27a() {
 	start_mds || return 2
 	echo "Requeue thread should have started: " 
 	ps -e | grep ll_cfg_requeue 
-	set_and_check "cat $LPROC/obdfilter/$FSNAME-OST0000/client_cache_seconds" "$FSNAME-OST0000.ost.client_cache_seconds" || return 3 
+	set_and_check ost1 "cat $LPROC/obdfilter/$FSNAME-OST0000/client_cache_seconds" "$FSNAME-OST0000.ost.client_cache_seconds" || return 3 
 	cleanup_nocli
 }
 run_test 27a "Reacquire MGS lock if OST started first"
@@ -936,8 +958,8 @@ run_test 27a "Reacquire MGS lock if OST started first"
 test_27b() {
         setup
 	facet_failover mds
-	set_and_check "cat $LPROC/mds/$FSNAME-MDT0000/group_acquire_expire" "$FSNAME-MDT0000.mdt.group_acquire_expire" || return 3 
-	set_and_check "cat $LPROC/mdc/$FSNAME-MDT0000-mdc-*/max_rpcs_in_flight" "$FSNAME-MDT0000.mdc.max_rpcs_in_flight" || return 4 
+	set_and_check mds "cat $LPROC/mdt/$FSNAME-MDT0000/identity_acquire_expire" "$FSNAME-MDT0000.mdt.identity_acquire_expire" || return 3
+	set_and_check client "cat $LPROC/mdc/$FSNAME-MDT0000-mdc-*/max_rpcs_in_flight" "$FSNAME-MDT0000.mdc.max_rpcs_in_flight" || return 4
 	cleanup
 }
 run_test 27b "Reacquire MGS lock after failover"
@@ -948,8 +970,8 @@ test_28() {
 	ORIG=$($TEST) 
 	declare -i FINAL
 	FINAL=$(($ORIG + 10))
-	set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
-	set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
+	set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
+	set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" || return 3
  	umount_client $MOUNT || return 200
 	mount_client $MOUNT
 	RESULT=$($TEST)
@@ -964,14 +986,14 @@ test_28() {
 run_test 28 "permanent parameter setting"
 
 test_29() {
-	[ "$OSTCOUNT" -lt "2" ] && echo "skipping deactivate test" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2, skipping" && return
         setup > /dev/null 2>&1
 	start_ost2
 	sleep 10
 
 	local PARAM="$FSNAME-OST0001.osc.active"
-	local PROC_ACT="$LPROC/osc/$FSNAME-OST0001-osc-*/active"
-	local PROC_UUID="$LPROC/osc/$FSNAME-OST0001-osc-*/ost_server_uuid"
+	local PROC_ACT="$LPROC/osc/$FSNAME-OST0001-osc-[^M]*/active"
+	local PROC_UUID="$LPROC/osc/$FSNAME-OST0001-osc-[^M]*/ost_server_uuid"
 	if [ ! -r $PROC_ACT ]; then
 	    echo "Can't read $PROC_ACT"
 	    ls $LPROC/osc/$FSNAME-*
@@ -979,7 +1001,7 @@ test_29() {
 	fi
 	ACTV=$(cat $PROC_ACT)
 	DEAC=$((1 - $ACTV))
-	set_and_check "cat $PROC_ACT" "$PARAM" $DEAC || return 2
+	set_and_check client "cat $PROC_ACT" "$PARAM" $DEAC || return 2
         # also check ost_server_uuid status
 	RESULT=$(grep DEACTIV $PROC_UUID)
 	if [ -z "$RESULT" ]; then
@@ -990,15 +1012,24 @@ test_29() {
 	fi
 
 	# check MDT too 
-	local MPROC="$LPROC/osc/$FSNAME-OST0001-osc/active"
-	if [ -r $MPROC ]; then
-	    RESULT=$(cat $MPROC)
-	    if [ $RESULT -ne $DEAC ]; then
-		echo "MDT not deactivated: $(cat $MPROC)"
+	local MPROC="$LPROC/osc/$FSNAME-OST0001-osc-[M]*/active"
+	local MAX=30
+	local WAIT=0
+	while [ 1 ]; do
+	    sleep 5
+	    RESULT=`do_facet mds " [ -r $MPROC ] && cat $MPROC"`
+	    [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $MPROC"
+	    if [ $RESULT -eq $DEAC ]; then
+		echo "MDT deactivated also after $WAIT sec (got $RESULT)"
+		break
+	    fi
+	    WAIT=$((WAIT + 5))
+	    if [ $WAIT -eq $MAX ]; then
+		echo "MDT not deactivated: wanted $DEAC got $RESULT"
 		return 4
 	    fi
-	    echo "MDT deactivated also"
-	fi
+	    echo "Waiting $(($MAX - $WAIT)) secs for MDT deactivated"
+	done
 
         # test new client starts deactivated
  	umount_client $MOUNT || return 200
@@ -1012,7 +1043,7 @@ test_29() {
 	fi
 
 	# make sure it reactivates
-	set_and_check "cat $PROC_ACT" "$PARAM" $ACTV || return 6
+	set_and_check client "cat $PROC_ACT" "$PARAM" $ACTV || return 6
 
  	umount_client $MOUNT
 	stop_ost2
@@ -1030,13 +1061,13 @@ test_30() {
 	TEST="cat $LPROC/llite/$FSNAME-*/max_read_ahead_whole_mb"
 	ORIG=$($TEST) 
 	for i in $(seq 1 20); do 
-	    set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $i || return 3
+	    set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $i || return 3
 	done
 	# make sure client restart still works 
  	umount_client $MOUNT
 	mount_client $MOUNT || return 4
 	[ "$($TEST)" -ne "$i" ] && return 5   
-	set_and_check "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $ORIG || return 6
+	set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $ORIG || return 6
 	cleanup
 }
 run_test 30 "Big config llog"
@@ -1048,8 +1079,334 @@ test_31() { # bug 10734
 }
 run_test 31 "Connect to non-existent node (shouldn't crash)"
 
+test_32a() {
+        # XXX - make this run on client-only systems with real hardware on
+        #       the OST and MDT
+        #       there appears to be a lot of assumption here about loopback
+        #       devices
+        # or maybe this test is just totally useless on a client-only system
+	[ "$mds_HOST" = "`hostname`" ] || { skip "remote MDS" && return 0; }
+	[ "$ost_HOST" = "`hostname`" -o "$ost1_HOST" = "`hostname`" ] || \
+		{ skip "remote OST" && return 0; }
+
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+	local DISK1_4=$LUSTRE/tests/disk1_4.zip
+        [ ! -r $DISK1_4 ] && skip "Cant find $DISK1_4, skipping" && return
+	unzip -o -j -d $TMP/$tdir $DISK1_4 || { skip "Cant unzip $DISK1_4, skipping" && return ; }
+	load_modules
+	sysctl lnet.debug=$PTLDEBUG
+
+	$TUNEFS $TMP/$tdir/mds || error "tunefs failed"
+	# nids are wrong, so client wont work, but server should start
+        start mds $TMP/$tdir/mds "-o loop,exclude=lustre-OST0000" || return 3
+        local UUID=$(cat $LPROC/mds/lustre-MDT0000/uuid)
+	echo MDS uuid $UUID
+	[ "$UUID" == "mdsA_UUID" ] || error "UUID is wrong: $UUID" 
+
+	$TUNEFS --mgsnode=`hostname` $TMP/$tdir/ost1 || error "tunefs failed"
+	start ost1 $TMP/$tdir/ost1 "-o loop" || return 5
+        UUID=$(cat $LPROC/obdfilter/lustre-OST0000/uuid)
+	echo OST uuid $UUID
+	[ "$UUID" == "ost1_UUID" ] || error "UUID is wrong: $UUID" 
+
+	local NID=$($LCTL list_nids | head -1)
+
+	echo "OSC changes should return err:" 
+	$LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 && return 7
+	$LCTL conf_param lustre-OST0000.failover.node=$NID && return 8
+	echo "ok."
+	echo "MDC changes should succeed:" 
+	$LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
+	$LCTL conf_param lustre-MDT0000.failover.node=$NID || return 10
+	echo "ok."
+
+	# With a new good MDT failover nid, we should be able to mount a client
+	# (but it cant talk to OST)
+        local OLDMOUNTOPT=$MOUNTOPT
+        MOUNTOPT="exclude=lustre-OST0000"
+	mount_client $MOUNT
+        MOUNTOPT=$OLDMOUNTOPT
+	set_and_check client "cat $LPROC/mdc/*/max_rpcs_in_flight" "lustre-MDT0000.mdc.max_rpcs_in_flight" || return 11
+
+	zconf_umount `hostname` $MOUNT -f
+	cleanup_nocli
+
+        # mount a second time to make sure we didnt leave upgrade flag on
+        $TUNEFS --dryrun $TMP/$tdir/mds || error "tunefs failed"
+        start mds $TMP/$tdir/mds "-o loop,exclude=lustre-OST0000" || return 12
+        cleanup_nocli
+
+	[ -d $TMP/$tdir ] && rm -rf $TMP/$tdir
+}
+run_test 32a "Upgrade from 1.4 (not live)"
+
+test_32b() {
+        # XXX - make this run on client-only systems with real hardware on
+        #       the OST and MDT
+        #       there appears to be a lot of assumption here about loopback
+        #       devices
+        # or maybe this test is just totally useless on a client-only system
+        [ "$mds_HOST" = "`hostname`" ] || { skip "remote MDS" && return 0; }
+        [ "$ost_HOST" = "`hostname`" -o "$ost1_HOST" = "`hostname`" ] || \
+		{ skip "remote OST" && return 0; }
+
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+	local DISK1_4=$LUSTRE/tests/disk1_4.zip
+        [ ! -r $DISK1_4 ] && skip "Cant find $DISK1_4, skipping" && return
+	unzip -o -j -d $TMP/$tdir $DISK1_4 || { skip "Cant unzip $DISK1_4, skipping" && return ; }
+	load_modules
+	sysctl lnet.debug=$PTLDEBUG
+
+	# writeconf will cause servers to register with their current nids
+	$TUNEFS --writeconf $TMP/$tdir/mds || error "tunefs failed"
+	start mds $TMP/$tdir/mds "-o loop" || return 3
+        local UUID=$(cat $LPROC/mds/lustre-MDT0000/uuid)
+	echo MDS uuid $UUID
+	[ "$UUID" == "mdsA_UUID" ] || error "UUID is wrong: $UUID" 
+
+	$TUNEFS --mgsnode=`hostname` $TMP/$tdir/ost1 || error "tunefs failed"
+	start ost1 $TMP/$tdir/ost1 "-o loop" || return 5
+        UUID=$(cat $LPROC/obdfilter/lustre-OST0000/uuid)
+	echo OST uuid $UUID
+	[ "$UUID" == "ost1_UUID" ] || error "UUID is wrong: $UUID" 
+
+	echo "OSC changes should succeed:" 
+	$LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 || return 7
+	$LCTL conf_param lustre-OST0000.failover.node=$NID || return 8
+	echo "ok."
+	echo "MDC changes should succeed:" 
+	$LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
+	echo "ok."
+
+	# MDT and OST should have registered with new nids, so we should have
+	# a fully-functioning client
+	echo "Check client and old fs contents"
+	mount_client $MOUNT
+	set_and_check client "cat $LPROC/mdc/*/max_rpcs_in_flight" "${NEWNAME}-MDT0000.mdc.max_rpcs_in_flight" || return 11
+	[ "$(cksum $MOUNT/passwd | cut -d' ' -f 1,2)" == "2479747619 779" ] || return 12  
+	echo "ok."
+
+	cleanup
+	[ -d $TMP/$tdir ] && rm -rf $TMP/$tdir
+}
+run_test 32b "Upgrade from 1.4 with writeconf"
+
+test_33() { # bug 12333
+        local FSNAME2=test1234
+        local fs2mds_HOST=$mds_HOST
+        local fs2ost_HOST=$ost_HOST
+        [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
+
+        if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" ]; then
+                do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
+                skip "mixed loopback and real device not working" && return
+        fi
+
+        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
+        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
+        add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --index=8191 --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
+
+        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
+        start fs2ost $fs2ostdev $OST_MOUNT_OPTS
+        mkdir -p $MOUNT2
+        mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
+        echo "ok."
+
+        umount -d $MOUNT2
+        stop fs2ost -f
+        stop fs2mds -f
+        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev
+        cleanup_nocli || return 6
+}
+run_test 33 "Mount ost with a large index number"
+
+test_34() {
+        setup
+
+        do_facet client dd if=/dev/zero of=$MOUNT/24 bs=1024k count=1
+        # Drop lock cancelation reply during umount
+	#define OBD_FAIL_LDLM_CANCEL             0x304
+        do_facet client sysctl -w lustre.fail_loc=0x80000304
+        #sysctl -w lnet.debug=-1
+        umount_client $MOUNT
+        cleanup
+}
+run_test 34 "Drop cancel during umount"
+
+test_34a() {
+        setup
+	do_facet client multiop $DIR/file O_c &
+	sleep 0.500s
+	manual_umount_client
+	rc=$?
+	do_facet client killall -USR1 multiop
+	if [ $rc -eq 0 ]; then
+		error "umount not fail!"
+	fi
+	sleep 1
+        cleanup
+}
+run_test 34a "umount with opened file should be fail"
+
+
+test_34b() {
+	setup
+	touch $DIR/$tfile || return 1
+	stop_mds --force || return 2
+
+ 	manual_umount_client --force
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		error "mtab after failed umount - rc $rc"
+	fi
+
+	cleanup
+	return 0	
+}
+run_test 34b "force umount with failed mds should be normal"
+
+test_34c() {
+	setup
+	touch $DIR/$tfile || return 1
+	stop_ost --force || return 2
+
+ 	manual_umount_client --force
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		error "mtab after failed umount - rc $rc"
+	fi
+
+	cleanup
+	return 0	
+}
+run_test 34c "force umount with failed ost should be normal"
+
+test_35() { # bug 12459
+	setup
+
+	DBG_SAVE="`sysctl -n lnet.debug`"
+	sysctl -w lnet.debug="ha"
+
+	log "Set up a fake failnode for the MDS"
+	FAKENID="127.0.0.2"
+	do_facet mds $LCTL conf_param ${FSNAME}-MDT0000.failover.node=$FAKENID || return 4
+
+	log "Wait for RECONNECT_INTERVAL seconds (10s)"
+	sleep 10
+
+	MSG="conf-sanity.sh test_35 `date +%F%kh%Mm%Ss`"
+	$LCTL clear
+	log "$MSG"
+	log "Stopping the MDT:"
+	stop_mds || return 5
+
+	df $MOUNT > /dev/null 2>&1 &
+	DFPID=$!
+	log "Restarting the MDT:"
+	start_mds || return 6
+	log "Wait for df ($DFPID) ... "
+	wait $DFPID
+	log "done"
+	sysctl -w lnet.debug="$DBG_SAVE"
+
+	# retrieve from the log the first server that the client tried to
+	# contact after the connection loss
+	$LCTL dk $TMP/lustre-log-$TESTNAME.log
+	NEXTCONN=`awk "/${MSG}/ {start = 1;}
+		       /import_select_connection.*${FSNAME}-MDT0000-mdc.* using connection/ {
+				if (start) {
+					if (\\\$NF ~ /$FAKENID/)
+						print \\\$NF;
+					else
+						print 0;
+					exit;
+				}
+		       }" $TMP/lustre-log-$TESTNAME.log`
+	[ "$NEXTCONN" != "0" ] && log "The client didn't try to reconnect to the last active server (tried ${NEXTCONN} instead)" && return 7
+	cleanup
+}
+run_test 35 "Reconnect to the last active server first"
+
+test_36() { # 12743
+        local rc
+        local FSNAME2=test1234
+        local fs2mds_HOST=$mds_HOST
+        local fs2ost_HOST=$ost_HOST
+        local fs3ost_HOST=$ost_HOST
+
+        [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST && fs3ost_HOST=$ost1_HOST
+        rc=0
+
+        if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" -o -z "$fs3ost_DEV" ]; then
+		do_facet $SINGLEMDS [ -b "$MDSDEV" ] && \
+		skip "mixed loopback and real device not working" && return
+        fi
+        [ $OSTCOUNT -lt 2 ] && skip "skipping test for single OST" && return
+
+        [ $(grep -c obdfilter $LPROC/devices) -eq 0 ] &&
+                skip "skipping test for remote OST" && return
+
+        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
+        local fs3ostdev=${fs3ost_DEV:-$(ostdevname 2)_2}
+        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
+        add fs2ost $OST_MKFS_OPTS --mkfsoptions='-b1024' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
+        add fs3ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs3ostdev || exit 10
+
+        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
+        start fs2ost $fs2ostdev $OST_MOUNT_OPTS
+        start fs3ost $fs3ostdev $OST_MOUNT_OPTS
+        mkdir -p $MOUNT2
+        mount -t lustre $MGSNID:/${FSNAME2} $MOUNT2 || return 1
+
+        sleep 5 # until 11778 fixed
+
+        dd if=/dev/zero of=$MOUNT2/$tfile bs=1M count=7 || return 2
+
+        BKTOTAL=`awk 'BEGIN{total=0}; {total+=$1}; END{print total}' \
+                $LPROC/obdfilter/*/kbytestotal`
+        BKFREE=`awk 'BEGIN{free=0}; {free+=$1}; END{print free}' \
+               $LPROC/obdfilter/*/kbytesfree`
+        BKAVAIL=`awk 'BEGIN{avail=0}; {avail+=$1}; END{print avail}' \
+                $LPROC/obdfilter/*/kbytesavail`
+        STRING=`df -P $MOUNT2 | tail -n 1 | awk '{print $2","$3","$4}'`
+        DFTOTAL=`echo $STRING | cut -d, -f1`
+        DFUSED=`echo $STRING  | cut -d, -f2`
+        DFAVAIL=`echo $STRING | cut -d, -f3`
+        DFFREE=$(($DFTOTAL - $DFUSED))
+
+        ALLOWANCE=$((64 * $OSTCOUNT))
+
+        if [ $DFTOTAL -lt $(($BKTOTAL - $ALLOWANCE)) ] ||  
+           [ $DFTOTAL -gt $(($BKTOTAL + $ALLOWANCE)) ] ; then
+                echo "**** FAIL: df total($DFTOTAL) mismatch OST total($BKTOTAL)"
+                rc=1
+        fi
+        if [ $DFFREE -lt $(($BKFREE - $ALLOWANCE)) ] || 
+           [ $DFFREE -gt $(($BKFREE + $ALLOWANCE)) ] ; then
+                echo "**** FAIL: df free($DFFREE) mismatch OST free($BKFREE)"
+                rc=2
+        fi
+        if [ $DFAVAIL -lt $(($BKAVAIL - $ALLOWANCE)) ] || 
+           [ $DFAVAIL -gt $(($BKAVAIL + $ALLOWANCE)) ] ; then
+                echo "**** FAIL: df avail($DFAVAIL) mismatch OST avail($BKAVAIL)"
+                rc=3
+       fi
+
+        umount -d $MOUNT2
+        stop fs3ost -f || return 200
+        stop fs2ost -f || return 201
+        stop fs2mds -f || return 202
+        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev $fs3ostdev
+        unload_modules || return 203
+        return $rc
+}
+run_test 36 "df report consistency on OSTs with different block size"
+
 umount_client $MOUNT	
 cleanup_nocli
+cleanup_krb5_env
 
-equals_msg "Done"
-echo "$0: completed"
+equals_msg `basename $0`: test complete
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true