X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fconf-sanity.sh;h=b55c9d3583c382082a17e01881c764e17cfa7813;hp=e0df8b8d26c96eb3d9993eb4deea111863011c56;hb=468de179b3a2c33df62454216de13bdb6f14614e;hpb=3b2d496affc294e86744ab9b1c1014f0ce81bb3e

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index e0df8b8..b55c9d3 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -13,15 +13,34 @@ ONLY=${ONLY:-"$*"}
 ALWAYS_EXCEPT="$CONF_SANITY_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
+is_sles11()						# LU-2181
+{
+	if [ -r /etc/SuSE-release ]
+	then
+		local vers=`grep VERSION /etc/SuSE-release | awk '{print $3}'`
+		local patchlev=`grep PATCHLEVEL /etc/SuSE-release \
+			| awk '{print $3}'`
+		if [ $vers -eq 11 ] && [ $patchlev -eq 2 ]
+		then
+			return 0
+		fi
+	fi
+	return 1
+}
+
+if is_sles11; then					# LU-2181
+	ALWAYS_EXCEPT="$ALWAYS_EXCEPT 23a 34b"
+fi
+
 if [ "$FAILURE_MODE" = "HARD" ]; then
 	CONFIG_EXCEPTIONS="24a " && \
 	echo "Except the tests: $CONFIG_EXCEPTIONS for FAILURE_MODE=$FAILURE_MODE, bug 23573" && \
 	ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
 fi
 
-# LU-2059
-ALWAYS_EXCEPT="$ALWAYS_EXCEPT 5d 19b 21b 27a"
-
+# bug number for skipped test:
+# a tool to create lustre filesystem images
+ALWAYS_EXCEPT="32newtarball $ALWAYS_EXCEPT"
 
 SRCDIR=`dirname $0`
 PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH
@@ -418,6 +437,9 @@ test_5d() {
 	grep " $MOUNT " /etc/mtab && \
 		error false "unexpected entry in mtab before mount" && return 10
 
+	[ "$(facet_fstype ost1)" = "zfs" ] &&
+		skip "LU-2059: no local config for ZFS OSTs" && return
+
 	local rc=0
 	start_ost
 	start_mds
@@ -657,6 +679,9 @@ test_19a() {
 run_test 19a "start/stop MDS without OSTs"
 
 test_19b() {
+	[ "$(facet_fstype ost1)" = "zfs" ] &&
+		skip "LU-2059: no local config for ZFS OSTs" && return
+
 	start_ost || return 1
 	stop_ost -f || return 2
 }
@@ -693,6 +718,9 @@ test_21a() {
 run_test 21a "start mds before ost, stop ost first"
 
 test_21b() {
+	[ "$(facet_fstype ost1)" = "zfs" ] &&
+		skip "LU-2059: no local config for ZFS OSTs" && return
+
         start_ost
 	start_mds
         wait_osc_import_state mds ost FULL
@@ -958,6 +986,9 @@ test_26() {
 run_test 26 "MDT startup failure cleans LOV (should return errs)"
 
 test_27a() {
+	[ "$(facet_fstype ost1)" = "zfs" ] &&
+		skip "LU-2059: no local config for ZFS OSTs" && return
+
 	start_ost || return 1
 	start_mds || return 2
 	echo "Requeue thread should have started: "
@@ -1149,202 +1180,439 @@ test_31() { # bug 10734
 }
 run_test 31 "Connect to non-existent node (shouldn't crash)"
 
-# Use these start32/stop32 fn instead of t-f start/stop fn,
-# for local devices, to skip global facet vars init
-stop32 () {
-	local facet=$1
-	shift
-	echo "Stopping local ${MOUNT%/*}/${facet} (opts:$@)"
-	umount -d $@ ${MOUNT%/*}/${facet}
-	losetup -a
-}
-
-start32 () {
-	local facet=$1
-	shift
-	local device=$1
-	shift
-	mkdir -p ${MOUNT%/*}/${facet}
-
-	echo "Starting local ${facet}: $@ $device ${MOUNT%/*}/${facet}"
-	mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}
-	local RC=$?
-	if [ $RC -ne 0 ]; then
-		echo "mount -t lustre $@ ${device} ${MOUNT%/*}/${facet}"
-		echo "Start of ${device} of local ${facet} failed ${RC}"
-	fi
-	losetup -a
-	return $RC
-}
-
-cleanup_nocli32 () {
-	stop32 mds1 -f
-	stop32 ost1 -f
-	wait_exit_ST client
-}
-
-cleanup_32() {
-	trap 0
-	echo "Cleanup test_32 umount $MOUNT ..."
-	umount -f $MOUNT || true
-	echo "Cleanup local mds ost1 ..."
-	cleanup_nocli32
-	combined_mgs_mds || start_mgs
-	unload_modules_conf
-}
-
-test_32a() {
-	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
-		skip "Only applicable to ldiskfs-based MDTs"
-		return
+#
+# This is not really a test but a tool to create new disk
+# image tarballs for the upgrade tests.
+#
+# Disk image tarballs should be created on single-node
+# clusters by running this test with default configurations
+# plus a few mandatory environment settings that are verified
+# at the beginning of the test.
+#
+test_32newtarball() {
+	local version
+	local dst=.
+	local src=/etc/rc.d
+	local tmp=$TMP/t32_image_create
+
+	if [ $FSNAME != t32fs -o $MDSCOUNT -ne 1 -o								\
+		 \( -z "$MDSDEV" -a -z "$MDSDEV1" \) -o $OSTCOUNT -ne 1 -o			\
+		 -z "$OSTDEV1" ]; then
+		error "Needs FSNAME=t32fs MDSCOUNT=1 MDSDEV1=<nonexistent_file>"	\
+			  "(or MDSDEV, in the case of b1_8) OSTCOUNT=1"					\
+			  "OSTDEV1=<nonexistent_file>"
 	fi
 
-	client_only && skip "client only testing" && return 0
-	[ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; }
-	[ -z "$TUNEFS" ] && skip_env "No tunefs" && return 0
+	mkdir $tmp || {
+		echo "Found stale $tmp"
+		return 1
+	}
 
-	local DISK1_8=$LUSTRE/tests/disk1_8.tar.bz2
-	[ ! -r $DISK1_8 ] && skip_env "Cannot find $DISK1_8" && return 0
-	local tmpdir=$TMP/conf32a
-	mkdir -p $tmpdir
-	tar xjvf $DISK1_8 -C $tmpdir || \
-		{ skip_env "Cannot untar $DISK1_8" && return 0; }
+	mkdir $tmp/src
+	tar cf - -C $src . | tar xf - -C $tmp/src
 
-	load_modules
-	$LCTL set_param debug="$PTLDEBUG"
+	formatall
 
-	$TUNEFS $tmpdir/mds || error "tunefs failed"
+	setupall
+	tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
+	stopall
 
-	combined_mgs_mds || stop mgs
+	mkdir $tmp/img
 
-	# nids are wrong, so client wont work, but server should start
-	start32 mds1 $tmpdir/mds "-o loop,exclude=lustre-OST0000" && \
-		trap cleanup_32 EXIT INT || return 3
+	setupall
+	pushd /mnt/$FSNAME
+	ls -Rni --time-style=+%s >$tmp/img/list
+	find . ! -name .lustre -type f -exec sha1sum {} \; |
+		sort -k 2 >$tmp/img/sha1sums
+	popd
+	$LCTL get_param -n version | head -n 1 |
+		sed -e 's/^lustre: *//' >$tmp/img/commit
+	stopall
 
-	local UUID=$($LCTL get_param -n mdt.lustre-MDT0000.uuid)
-	echo MDS uuid $UUID
-	[ "$UUID" == "lustre-MDT0000_UUID" ] || error "UUID is wrong: $UUID"
+	pushd $tmp/src
+	find -type f -exec sha1sum {} \; | sort -k 2 >$tmp/sha1sums.src
+	popd
 
-	$TUNEFS --mgsnode=$HOSTNAME $tmpdir/ost1 || error "tunefs failed"
-	start32 ost1 $tmpdir/ost1 "-o loop" || return 5
-	UUID=$($LCTL get_param -n obdfilter.lustre-OST0000.uuid)
-	echo OST uuid $UUID
-	[ "$UUID" == "lustre-OST0000_UUID" ] || error "UUID is wrong: $UUID"
+	if ! diff -u $tmp/sha1sums.src $tmp/img/sha1sums; then
+		echo "Data verification failed"
+	fi
 
-	local NID=$($LCTL list_nids | head -1)
+	uname -r >$tmp/img/kernel
+	uname -m >$tmp/img/arch
 
-	echo "OSC changes should succeed:"
-	$LCTL conf_param lustre-OST0000.osc.max_dirty_mb=15 || return 7
-	$LCTL conf_param lustre-OST0000.failover.node=$NID || return 8
-	echo "ok."
+	mv ${MDSDEV1:-$MDSDEV} $tmp/img
+	mv $OSTDEV1 $tmp/img
 
-	echo "MDC changes should succeed:"
-	$LCTL conf_param lustre-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
-	$LCTL conf_param lustre-MDT0000.failover.node=$NID || return 10
-	echo "ok."
+	version=$(sed -e 's/\(^[0-9]\+\.[0-9]\+\)\(.*$\)/\1/' $tmp/img/commit |
+			  sed -e 's/\./_/g')	# E.g., "1.8.7" -> "1_8"
+	dst=$(cd $dst; pwd)
+	pushd $tmp/img
+	tar cjvf $dst/disk$version-$(facet_fstype $SINGLEMDS).tar.bz2 -S *
+	popd
 
-	echo "LOV changes should succeed:"
-	$LCTL pool_new lustre.interop || return 11
-	$LCTL conf_param lustre-MDT0000.lov.stripesize=4M || return 12
-	echo "ok."
+	rm -r $tmp
+}
+#run_test 32newtarball "Create a new test_32 disk image tarball for this version"
 
-	cleanup_32
+#
+# The list of applicable tarballs is returned via the caller's
+# variable "tarballs".
+#
+t32_check() {
+	local node=$(facet_active_host $SINGLEMDS)
+	local r="do_node $node"
 
-	# mount a second time to make sure we didnt leave upgrade flag on
-	load_modules
-	$TUNEFS --dryrun $tmpdir/mds || error "tunefs failed"
+	if [ "$CLIENTONLY" ]; then
+		skip "Client-only testing"
+		exit 0
+	fi
 
-	combined_mgs_mds || stop mgs
+	if ! $r which $TUNEFS; then
+		skip_env "tunefs.lustre required on $node"
+		exit 0
+	fi
 
-	start32 mds1 $tmpdir/mds "-o loop,exclude=lustre-OST0000" && \
-		trap cleanup_32 EXIT INT || return 12
+	if [ -n "$($LCTL list_nids | grep -v '\(tcp\|lo\)[[:digit:]]*$')" ]; then
+		skip "LU-2200: Test cannot run over Infiniband"
+		exit 0
+	fi
 
-	cleanup_32
+	local IMGTYPE=$(facet_fstype $SINGLEMDS)
 
-	rm -rf $tmpdir || true	# true is only for TMP on NFS
-}
-run_test 32a "Upgrade from 1.8 (not live)"
+	tarballs=$($r find $RLUSTRE/tests -maxdepth 1 -name \'disk*-$IMGTYPE.tar.bz2\')
 
-test_32b() {
-	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
-		skip "Only applicable to ldiskfs-based MDTs"
-		return
+	if [ -z "$tarballs" ]; then
+		skip "No applicable tarballs found"
+		exit 0
 	fi
+}
 
-	client_only && skip "client only testing" && return 0
-	[ "$NETTYPE" = "tcp" ] || { skip "NETTYPE != tcp" && return 0; }
-	[ -z "$TUNEFS" ] && skip_env "No tunefs" && return
+t32_test_cleanup() {
+	local node=$(facet_active_host $SINGLEMDS)
+	local r="do_node $node"
+	local tmp=$TMP/t32
+	local rc=$?
 
-	local DISK1_8=$LUSTRE/tests/disk1_8.tar.bz2
-	[ ! -r $DISK1_8 ] && skip_env "Cannot find $DISK1_8" && return 0
-	local tmpdir=$TMP/conf32b
-	mkdir -p $tmpdir
-	tar xjvf $DISK1_8 -C $tmpdir || \
-		{ skip_env "Cannot untar $DISK1_8" && return ; }
+	if $shall_cleanup_lustre; then
+		umount $tmp/mnt/lustre || rc=$?
+	fi
+	if $shall_cleanup_mdt; then
+		$r umount -d $tmp/mnt/mdt || rc=$?
+	fi
+	if $shall_cleanup_ost; then
+		$r umount -d $tmp/mnt/ost || rc=$?
+	fi
+	$r rm -rf $tmp || rc=$?
+	rm -rf $tmp || rc=$?
+	return $rc
+}
 
-	load_modules
-	$LCTL set_param debug="+config"
-	local NEWNAME=lustre
+t32_bits_per_long() {
+	#
+	# Yes, this is not meant to be perfect.
+	#
+	case $1 in
+		ppc64|x86_64)
+			echo -n 64;;
+		i*86)
+			echo -n 32;;
+	esac
+}
 
-	# writeconf will cause servers to register with their current nids
-	$TUNEFS --writeconf --erase-params \
-        --param mdt.identity_upcall=$L_GETIDENTITY \
-        --fsname=$NEWNAME $tmpdir/mds || error "tunefs failed"
-	combined_mgs_mds || stop mgs
+t32_reload_modules() {
+	local node=$1
+	local all_removed=false
+	local i=0
+
+	while ((i < 20)); do
+		echo "Unloading modules on $node: Attempt $i"
+		do_rpc_nodes $node $LUSTRE_RMMOD $(facet_fstype $SINGLEMDS) &&
+			all_removed=true
+		do_rpc_nodes $node check_mem_leak || return 1
+		if $all_removed; then
+			load_modules
+			return 0
+		fi
+		sleep 5
+		i=$((i + 1))
+	done
+	echo "Unloading modules on $node: Given up"
+	return 1
+}
 
-	start32 mds1 $tmpdir/mds "-o loop" && \
-		trap cleanup_32 EXIT INT || return 3
+t32_wait_til_devices_gone() {
+	local node=$1
+	local devices
+	local i=0
+
+	echo wait for devices to go
+	while ((i < 20)); do
+		devices=$(do_rpc_nodes $node $LCTL device_list | wc -l)
+		echo $device
+		((devices == 0)) && return 1
+		sleep 5
+		i=$((i + 1))
+	done
+	echo "waiting for devices on $node: Given up"
+	return 1
+}
+
+t32_test() {
+	local tarball=$1
+	local writeconf=$2
+	local shall_cleanup_mdt=false
+	local shall_cleanup_ost=false
+	local shall_cleanup_lustre=false
+	local node=$(facet_active_host $SINGLEMDS)
+	local r="do_node $node"
+	local tmp=$TMP/t32
+	local img_commit
+	local img_kernel
+	local img_arch
+	local fsname=t32fs
+	local nid=$($r $LCTL list_nids | head -1)
+	local mopts
+	local uuid
+	local nrpcs_orig
+	local nrpcs
+	local list
+
+	trap 'trap - RETURN; t32_test_cleanup' RETURN
+
+	mkdir -p $tmp/mnt/lustre
+	$r mkdir -p $tmp/mnt/{mdt,ost}
+	$r tar xjvf $tarball -S -C $tmp || {
+		error_noexit "Unpacking the disk image tarball"
+		return 1
+	}
+	img_commit=$($r cat $tmp/commit)
+	img_kernel=$($r cat $tmp/kernel)
+	img_arch=$($r cat $tmp/arch)
+	echo "Upgrading from $(basename $tarball), created with:"
+	echo "  Commit: $img_commit"
+	echo "  Kernel: $img_kernel"
+	echo "    Arch: $img_arch"
+
+	$r $LCTL set_param debug="$PTLDEBUG"
+
+	$r $TUNEFS --dryrun $tmp/mdt || {
+		error_noexit "tunefs.lustre before mounting the MDT"
+		return 1
+	}
+	if [ "$writeconf" ]; then
+		mopts=loop,writeconf
+	else
+		mopts=loop,exclude=$fsname-OST0000
+	fi
 
-	local UUID=$($LCTL get_param -n mdt.${NEWNAME}-MDT0000.uuid)
-	echo MDS uuid $UUID
-	[ "$UUID" == "${NEWNAME}-MDT0000_UUID" ] || error "UUID is wrong: $UUID"
+	t32_wait_til_devices_gone $node
 
-	$TUNEFS  --writeconf --erase-params \
-        --mgsnode=$HOSTNAME --fsname=$NEWNAME $tmpdir/ost1 ||\
-	    error "tunefs failed"
-	start32 ost1 $tmpdir/ost1 "-o loop" || return 5
-	UUID=$($LCTL get_param -n obdfilter.${NEWNAME}-OST0000.uuid)
-	echo OST uuid $UUID
-	[ "$UUID" == "${NEWNAME}-OST0000_UUID" ] || error "UUID is wrong: $UUID"
+	$r mount -t lustre -o $mopts $tmp/mdt $tmp/mnt/mdt || {
+		error_noexit "Mounting the MDT"
+		return 1
+	}
+	shall_cleanup_mdt=true
 
-	local NID=$($LCTL list_nids | head -1)
+	uuid=$($r $LCTL get_param -n mdt.$fsname-MDT0000.uuid) || {
+		error_noexit "Getting MDT UUID"
+		return 1
+	}
+	if [ "$uuid" != $fsname-MDT0000_UUID ]; then
+		error_noexit "Unexpected MDT UUID: \"$uuid\""
+		return 1
+	fi
 
-	echo "OSC changes should succeed:"
-	$LCTL conf_param ${NEWNAME}-OST0000.osc.max_dirty_mb=15 || return 7
-	$LCTL conf_param ${NEWNAME}-OST0000.failover.node=$NID || return 8
-	echo "ok."
+	$r $TUNEFS --dryrun $tmp/ost || {
+		error_noexit "tunefs.lustre before mounting the OST"
+		return 1
+	}
+	if [ "$writeconf" ]; then
+		mopts=loop,mgsnode=$nid,$writeconf
+	else
+		mopts=loop,mgsnode=$nid
+	fi
+	$r mount -t lustre -o $mopts $tmp/ost $tmp/mnt/ost || {
+		error_noexit "Mounting the OST"
+		return 1
+	}
+	shall_cleanup_ost=true
 
-	echo "MDC changes should succeed:"
-	$LCTL conf_param ${NEWNAME}-MDT0000.mdc.max_rpcs_in_flight=9 || return 9
-	$LCTL conf_param ${NEWNAME}-MDT0000.failover.node=$NID || return 10
-	echo "ok."
+	uuid=$($r $LCTL get_param -n obdfilter.$fsname-OST0000.uuid) || {
+		error_noexit "Getting OST UUID"
+		return 1
+	}
+	if [ "$uuid" != $fsname-OST0000_UUID ]; then
+		error_noexit "Unexpected OST UUID: \"$uuid\""
+		return 1
+	fi
 
-	echo "LOV changes should succeed:"
-	$LCTL pool_new ${NEWNAME}.interop || return 11
-	$LCTL conf_param ${NEWNAME}-MDT0000.lov.stripesize=4M || return 12
-	echo "ok."
+	$r $LCTL conf_param $fsname-OST0000.osc.max_dirty_mb=15 || {
+		error_noexit "Setting \"max_dirty_mb\""
+		return 1
+	}
+	$r $LCTL conf_param $fsname-OST0000.failover.node=$nid || {
+		error_noexit "Setting OST \"failover.node\""
+		return 1
+	}
+	$r $LCTL conf_param $fsname-MDT0000.mdc.max_rpcs_in_flight=9 || {
+		error_noexit "Setting \"max_rpcs_in_flight\""
+		return 1
+	}
+	$r $LCTL conf_param $fsname-MDT0000.failover.node=$nid || {
+		error_noexit "Setting MDT \"failover.node\""
+		return 1
+	}
+	$r $LCTL pool_new $fsname.interop || {
+		error_noexit "Setting \"interop\""
+		return 1
+	}
+	$r $LCTL conf_param $fsname-MDT0000.lov.stripesize=4M || {
+		error_noexit "Setting \"lov.stripesize\""
+		return 1
+	}
+
+	if [ "$writeconf" ]; then
+		mount -t lustre $nid:/$fsname $tmp/mnt/lustre || {
+			error_noexit "Mounting the client"
+			return 1
+		}
+		shall_cleanup_lustre=true
+		$LCTL set_param debug="$PTLDEBUG"
+
+		if $r test -f $tmp/sha1sums; then
+			$r sort -k 2 $tmp/sha1sums >$tmp/sha1sums.orig
+			pushd $tmp/mnt/lustre
+			find ! -name .lustre -type f -exec sha1sum {} \; |
+				sort -k 2 >$tmp/sha1sums || {
+				error_noexit "sha1sum"
+				return 1
+			}
+			popd
+			if ! diff -ub $tmp/sha1sums.orig $tmp/sha1sums; then
+				error_noexit "sha1sum verification failed"
+				return 1
+			fi
+		else
+			echo "sha1sum verification skipped"
+		fi
 
-	# MDT and OST should have registered with new nids, so we should have
-	# a fully-functioning client
-	echo "Check client and old fs contents"
+		if $r test -f $tmp/list; then
+			#
+			# There is not a Test Framework API to copy files to or
+			# from a remote node.
+			#
+			$r sort -k 6 $tmp/list >$tmp/list.orig
+			pushd $tmp/mnt/lustre
+			ls -Rni --time-style=+%s | sort -k 6 >$tmp/list || {
+				error_noexit "ls"
+				return 1
+			}
+			popd
+			#
+			# 32-bit and 64-bit clients use different algorithms to
+			# convert FIDs into inode numbers.  Hence, remove the inode
+			# numbers from the lists, if the original list was created
+			# on an architecture with different number of bits per
+			# "long".
+			#
+			if [ $(t32_bits_per_long $(uname -m)) !=						\
+				 $(t32_bits_per_long $img_arch) ]; then
+				echo "Different number of bits per \"long\" from the disk image"
+				for list in list.orig list; do
+					sed -i -e 's/^[0-9]\+[ \t]\+//' $tmp/$list
+				done
+			fi
+			if ! diff -ub $tmp/list.orig $tmp/list; then
+				error_noexit "list verification failed"
+				return 1
+			fi
+		else
+			echo "list verification skipped"
+		fi
 
-	local device=`h2$NETTYPE $HOSTNAME`:/$NEWNAME
-	echo "Starting local client: $HOSTNAME: $device $MOUNT"
-	mount -t lustre $device $MOUNT || return 1
+		#
+		# When adding new data verification tests, please check for
+		# the presence of the required reference files first, like
+		# the "sha1sums" and "list" tests above, to avoid the need to
+		# regenerate every image for each test addition.
+		#
+
+		nrpcs_orig=$($LCTL get_param -n mdc.*.max_rpcs_in_flight) || {
+			error_noexit "Getting \"max_rpcs_in_flight\""
+			return 1
+		}
+		nrpcs=$((nrpcs_orig + 5))
+		$r $LCTL conf_param $fsname-MDT0000.mdc.max_rpcs_in_flight=$nrpcs || {
+			error_noexit "Changing \"max_rpcs_in_flight\""
+			return 1
+		}
+		wait_update $HOSTNAME "$LCTL get_param -n mdc.*.max_rpcs_in_flight"	\
+		            $nrpcs || {
+			error_noexit "Verifying \"max_rpcs_in_flight\""
+			return 1
+		}
+
+		umount $tmp/mnt/lustre || {
+			error_noexit "Unmounting the client"
+			return 1
+		}
+		shall_cleanup_lustre=false
+	else
+		$r umount -d $tmp/mnt/mdt || {
+			error_noexit "Unmounting the MDT"
+			return 1
+		}
+		shall_cleanup_mdt=false
+		$r umount -d $tmp/mnt/ost || {
+			error_noexit "Unmounting the OST"
+			return 1
+		}
+		shall_cleanup_ost=false
+
+		t32_reload_modules $node || {
+			error_noexit "Reloading modules"
+			return 1
+		}
+
+		# mount a second time to make sure we didnt leave upgrade flag on
+		$r $TUNEFS --dryrun $tmp/mdt || {
+			error_noexit "tunefs.lustre before remounting the MDT"
+			return 1
+		}
+		$r mount -t lustre -o loop,exclude=$fsname-OST0000 $tmp/mdt			\
+				 $tmp/mnt/mdt || {
+			error_noexit "Remounting the MDT"
+			return 1
+		}
+		shall_cleanup_mdt=true
+	fi
+}
 
-	local old=$($LCTL get_param -n mdc.*.max_rpcs_in_flight)
-	local new=$((old + 5))
-	$LCTL conf_param ${NEWNAME}-MDT0000.mdc.max_rpcs_in_flight=$new
-	wait_update $HOSTNAME "$LCTL get_param -n mdc.*.max_rpcs_in_flight" $new || return 11
+test_32a() {
+	local tarballs
+	local tarball
+	local rc=0
 
-	[ "$(cksum $MOUNT/passwd | cut -d' ' -f 1,2)" == "94306271 1478" ] || return 12
-	echo "ok."
+	t32_check
+	for tarball in $tarballs; do
+		t32_test $tarball || rc=$?
+	done
+	return $rc
+}
+run_test 32a "Upgrade (not live)"
 
-	cleanup_32
+test_32b() {
+	local tarballs
+	local tarball
+	local rc=0
 
-	rm -rf $tmpdir || true  # true is only for TMP on NFS
+	t32_check
+	for tarball in $tarballs; do
+		t32_test $tarball writeconf || rc=$?
+	done
+	return $rc
 }
-run_test 32b "Upgrade from 1.8 with writeconf"
+run_test 32b "Upgrade with writeconf"
 
 test_33a() { # bug 12333, was test_33
         local rc=0
@@ -1398,7 +1666,7 @@ test_33b() {	# was test_34
 
         do_facet client dd if=/dev/zero of=$MOUNT/24 bs=1024k count=1
         # Drop lock cancelation reply during umount
-	#define OBD_FAIL_LDLM_CANCEL             0x304
+	#define OBD_FAIL_LDLM_CANCEL_NET			0x304
         do_facet client lctl set_param fail_loc=0x80000304
         #lctl set_param debug=-1
         umount_client $MOUNT
@@ -1462,7 +1730,8 @@ test_35a() { # bug 12459
 	log "Set up a fake failnode for the MDS"
 	FAKENID="127.0.0.2"
 	local device=$(do_facet $SINGLEMDS "lctl get_param -n devices" | awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
-	do_facet mgs $LCTL conf_param ${device}.failover.node=$FAKENID || return 4
+	do_facet mgs "$LCTL conf_param ${device}.failover.node=" \
+		"$(h2$NETTYPE $FAKENID)" || return 4
 
 	log "Wait for RECONNECT_INTERVAL seconds (10s)"
 	sleep 10
@@ -1516,8 +1785,8 @@ test_35b() { # bug 18674
 	FAKENID="127.0.0.2"
 	local device=$(do_facet $SINGLEMDS "$LCTL get_param -n devices" | \
 			awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
-	do_facet mgs "$LCTL conf_param ${device}.failover.node=$FAKENID" || \
-		return 1
+	do_facet mgs "$LCTL conf_param ${device}.failover.node=" \
+		"$(h2$NETTYPE $FAKENID)" || return 1
 
 	local at_max_saved=0
 	# adaptive timeouts may prevent seeing the issue
@@ -2650,9 +2919,6 @@ thread_sanity() {
         lassert 28 "$msg" '(($tstarted == $tmin && $tstarted == $tmax ))' || return $?
         cleanup
 
-        # Workaround a YALA bug where YALA expects that modules will remain
-        # loaded on the servers
-        LOAD_MODULES_REMOTE=false
         load_modules
         setup
         cleanup
@@ -2989,5 +3255,5 @@ fi
 
 cleanup_gss
 
-complete $(basename $0) $SECONDS
+complete $SECONDS
 exit_status