X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fconf-sanity.sh;h=7a9a105e600af71dddb2c63d99b791b73b14a821;hp=4599e5eaf67fb10ddf66df54c4a948911e9bed17;hb=e26d7cc3992252e5fce5a51aee716f933b04c13a;hpb=6b979daaffc36aeef145316b41d0e2fe8abcf20f

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 4599e5e..7a9a105 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -15,14 +15,12 @@ init_logging
 ALWAYS_EXCEPT="$CONF_SANITY_EXCEPT 32newtarball"
 
 # bug number for skipped test: LU-11915
-ALWAYS_EXCEPT="$ALWAYS_EXCEPT  110 115"
+ALWAYS_EXCEPT="$ALWAYS_EXCEPT 110"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 if $SHARED_KEY; then
 	# bug number for skipped tests: LU-9795	(all below)
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT	0	31	32a	32d	35a"
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT	53a	53b	54b	76a	76b"
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT	76c	76d	78	103"
+	ALWAYS_EXCEPT="$ALWAYS_EXCEPT   84      86      103"
 fi
 
 if ! combined_mgs_mds; then
@@ -71,7 +69,7 @@ gen_config() {
 	# and stop to generate the startup logs.
 	start_mds
 	start_ost
-        wait_osc_import_state mds ost FULL
+	wait_osc_import_state mds ost FULL
 	stop_ost
 	stop_mds
 }
@@ -95,7 +93,7 @@ writeconf_or_reformat() {
 }
 
 reformat() {
-        formatall
+	formatall
 }
 
 start_mgs () {
@@ -140,6 +138,9 @@ start_mds() {
 	for num in $(seq $MDSCOUNT); do
 		start_mdt $num $@ || return 94
 	done
+	for num in $(seq $MDSCOUNT); do
+		wait_clients_import_state ${CLIENTS:-$HOSTNAME} mds${num} FULL
+	done
 }
 
 start_mgsmds() {
@@ -157,14 +158,15 @@ stop_mds() {
 }
 
 stop_mgs() {
-       echo "stop mgs service on `facet_active_host mgs`"
-       # These tests all use non-failover stop
-       stop mgs -f  || return 97
+	echo "stop mgs service on `facet_active_host mgs`"
+	# These tests all use non-failover stop
+	stop mgs -f  || return 97
 }
 
 start_ost() {
 	echo "start ost1 service on `facet_active_host ost1`"
 	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS $@ || return 95
+	wait_clients_import_state ${CLIENTS:-$HOSTNAME} ost1 FULL
 }
 
 stop_ost() {
@@ -176,6 +178,7 @@ stop_ost() {
 start_ost2() {
 	echo "start ost2 service on `facet_active_host ost2`"
 	start ost2 $(ostdevname 2) $OST_MOUNT_OPTS $@ || return 92
+	wait_clients_import_state ${CLIENTS:-$HOSTNAME} ost2 FULL
 }
 
 stop_ost2() {
@@ -254,8 +257,8 @@ check_mount() {
 	do_facet client "cp /etc/passwd $DIR/a" || return 71
 	do_facet client "rm $DIR/a" || return 72
 	# make sure lustre is actually mounted (touch will block,
-        # but grep won't, so do it after)
-        do_facet client "grep $MOUNT' ' /proc/mounts > /dev/null" || return 73
+	# but grep won't, so do it after)
+	do_facet client "grep $MOUNT' ' /proc/mounts > /dev/null" || return 73
 	echo "setup single mount lustre success"
 }
 
@@ -527,6 +530,9 @@ test_5h() {
 run_test 5h "start mdt failure at mdt_fs_setup()"
 
 test_5i() {
+	(( $MDS1_VERSION >= $(version_code 2.12.54) )) ||
+		skip "Need MDS version at least 2.12.54"
+
 	setup
 
 	stop mds1
@@ -590,6 +596,27 @@ test_9() {
 }
 run_test 9 "test ptldebug and subsystem for mkfs"
 
+test_10a() {
+	setup
+
+	if ! combined_mgs_mds; then
+		files=$(do_facet mgs "find /{proc/fs,sys/fs,sys/kernel/debug}/lustre -type l -exec test ! -e {} \; -print")
+		[ -z $files ] || echo "MGS $files is a broken symlink"
+	fi
+
+	files=$(do_facet mds1 "find /{proc/fs,sys/fs,sys/kernel/debug}/lustre -type l -exec test ! -e {} \; -print")
+	[ -z $files ] || echo "MDS $files is a broken symlink"
+
+	files=$(do_facet ost1 "find /{proc/fs,sys/fs,sys/kernel/debug}/lustre -type l -exec test ! -e {} \; -print")
+	[ -z $files ] || echo "OSS $files is a broken symlink"
+
+	files=$(do_facet client "find /{proc/fs,sys/fs,sys/kernel/debug}/lustre -type l -exec test ! -e {} \; -print")
+	[ -z $files ] || echo "clients $files is a broken symlink"
+
+	cleanup || error "cleanup failed with rc $?"
+}
+run_test 10a "find lctl param broken symlinks"
+
 #
 # Test 16 was to "verify that lustre will correct the mode of OBJECTS".
 # But with new MDS stack we don't care about the mode of local objects
@@ -1335,8 +1362,7 @@ run_test 31 "Connect to non-existent node (shouldn't crash)"
 T32_QID=60000
 T32_BLIMIT=40960 # Kbytes
 T32_ILIMIT=4
-T32_PRJID=1000
-T32_PROLIMIT=$((T32_BLIMIT/10))
+
 #
 # This is not really a test but a tool to create new disk
 # image tarballs for the upgrade tests.
@@ -1354,24 +1380,14 @@ test_32newtarball() {
 	local remote_dir
 	local striped_dir
 	local pushd_dir
-	local pfl_dir
-	local pfl_file
-	local dom_dir
-	local dom_file
-	local flr_dir
-	local flr_file
-	local pj_quota_dir
-	local pj_quota_file
-	local target_dir
 
 	if [ $FSNAME != t32fs -o \( -z "$MDSDEV" -a -z "$MDSDEV1" \) -o	\
-	$OSTCOUNT -ne 2 -o -z "$OSTDEV1" ]; then
-		error   "Needs FSNAME=t32fs MDSCOUNT=2 "		\
-			"MDSDEV1=<nonexistent_file> "			\
-			"MDSDEV2=<nonexistent_file> "			\
-			"(or MDSDEV, in the case of b1_8) "		\
-			"OSTCOUNT=2 OSTDEV1=<nonexistent_file> "	\
-			"OSTDEV2=<nonexistent_file>"
+	     $OSTCOUNT -ne 1 -o	-z "$OSTDEV1" ]; then
+		error "Needs FSNAME=t32fs MDSCOUNT=2 "			\
+		      "MDSDEV1=<nonexistent_file>"			\
+		      "MDSDEV2=<nonexistent_file>"			\
+		      "(or MDSDEV, in the case of b1_8)"		\
+		      "OSTCOUNT=1 OSTDEV1=<nonexistent_file>"
 	fi
 
 	mkdir $tmp || {
@@ -1381,6 +1397,9 @@ test_32newtarball() {
 
 	mkdir $tmp/src || return 1
 	tar cf - -C $src . | tar xf - -C $tmp/src
+	dd if=/dev/zero of=$tmp/src/t32_qf_old bs=1M \
+		count=$(($T32_BLIMIT / 1024 / 4))
+	chown $T32_QID.$T32_QID $tmp/src/t32_qf_old
 
 	# format ost with comma-separated NIDs to verify LU-4460
 	local failnid="$(h2nettype 1.2.3.4),$(h2nettype 4.3.2.1)"
@@ -1393,104 +1412,20 @@ test_32newtarball() {
 	$LFS setquota -u $T32_QID -b 0 -B $T32_BLIMIT -i 0 -I $T32_ILIMIT \
 		/mnt/$FSNAME
 
+	tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
+
 	if [[ $MDSCOUNT -ge 2 ]]; then
 		remote_dir=/mnt/$FSNAME/remote_dir
 		$LFS mkdir -i 1 $remote_dir
 		tar cf - -C $tmp/src . | tar xf - -C $remote_dir
 
-		target_dir=$remote_dir
-		if [[ $MDS1_VERSION -ge $(version_code 2.7.0) ]]; then
+		if [[ "$MDS1_VERSION" -ge $(version_code 2.7.0) ]]; then
 			striped_dir=/mnt/$FSNAME/striped_dir_old
 			$LFS mkdir -i 1 -c 2 $striped_dir
 			tar cf - -C $tmp/src . | tar xf - -C $striped_dir
 		fi
-	else
-		tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
-		target_dir=/mnt/$FSNAME
 	fi
 
-	# add project quota #
-	[[ $mds1_FSTYPE == "ldiskfs" &&
-		$MDS1_VERSION -gt $(version_code 2.9.55) ]] ||
-	[[ $mds1_FSTYPE == "zfs" &&
-		$MDS1_VERSION -gt $(version_code 2.10.53) ]] && {
-		pj_quota_dir=$target_dir/project_quota_dir
-		pj_quota_file_old=$pj_quota_dir/pj_quota_file_old
-
-		enable_project_quota
-		set_mdt_qtype ugp ||
-			error "enable mdt quota failed"
-
-		set_ost_qtype ugp ||
-			error "enable ost quota failed"
-
-		mkdir -p $pj_quota_dir
-		$LFS setquota -p $T32_PRJID -b 0 -B $T32_PROLIMIT -i 0 \
-			-I $T32_ILIMIT $pj_quota_dir ||
-			error "setquota -p $T32_PRJID failed"
-
-		$LFS setstripe $pj_quota_file_old -c 1 ||
-			error "setstripe $pj_quota_file_old failed"
-
-		chown $T32_QID:$T32_QID $pj_quota_file_old ||
-			error "chown $pj_quota_file_old failed"
-
-		change_project -p $T32_PRJID $pj_quota_file_old
-
-		mkdir -p $tmp/src/project_quota_dir
-		cp $pj_quota_file_old $tmp/src/project_quota_dir/
-	}
-
-	#####################
-	tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
-
-	#if [[ $MDSCOUNT -ge 2 ]]; then
-	#	remote_dir=/mnt/$FSNAME/remote_dir
-	#	$LFS mkdir -i 1 $remote_dir
-	#	tar cf - -C $tmp/src . | tar xf - -C $remote_dir
-
-	#	if [[ "$MDS1_VERSION" -ge $(version_code 2.7.0) ]]; then
-	#		striped_dir=/mnt/$FSNAME/striped_dir_old
-	#		$LFS mkdir -i 1 -c 2 $striped_dir
-	#		tar cf - -C $tmp/src . | tar xf - -C $striped_dir
-	#	fi
-	#fi
-
-	# PFL file #
-	if [[ $MDS1_VERSION -ge $(version_code 2.9.51) ]]; then
-		pfl_dir=$target_dir/pfl_dir
-		pfl_file=$pfl_dir/pfl_file
-		mkdir -p $pfl_dir
-		$LFS setstripe -E 2M -c 1 -o 0 -E -1 -S 2M -c 1 -o 1 \
-					$pfl_file ||
-			error "Create PFL file failed"
-
-		dd if=/dev/urandom of=$pfl_file bs=1k count=3k
-		mkdir -p $tmp/src/pfl_dir
-		cp $pfl_file $tmp/src/pfl_dir/
-	fi
-
-	############
-	# DoM / FLR file #
-	if [[ $MDS1_VERSION -ge $(version_code 2.10.56) ]]; then
-		dom_dir=$target_dir/dom_dir
-		dom_file=$dom_dir/dom_file
-		flr_dir=$target_dir/flr_dir
-		flr_file=$flr_dir/flr_file
-
-		mkdir -p $dom_dir
-		$LFS setstripe -E 1M -L mdt -E -1 -S 4M $dom_file
-		dd if=/dev/urandom of=$dom_file bs=1k count=2k
-		mkdir -p $tmp/src/dom_dir
-		cp $dom_file $tmp/src/dom_dir
-	# FLR #
-		mkdir -p $flr_dir
-		LFS mirror create -N2 $flr_file
-		dd if=/dev/urandom of=$flr_file bs=1k count=1
-		mkdir -p $tmp/src/flr_dir
-		cp $flr_file $tmp/src/flr_dir
-	fi
-	############
 	stopall
 
 	mkdir $tmp/img || return 1
@@ -1539,7 +1474,6 @@ test_32newtarball() {
 	kill -s USR1 $pid
 	wait $pid
 
-	#################
 	stopall
 
 	pushd $tmp/src
@@ -1560,14 +1494,7 @@ test_32newtarball() {
 			devname=$(mdsvdevname $num)
 		mv $devname $tmp/img
 	done
-
-	for num in $(seq $OSTCOUNT); do
-		local devname=$(ostdevname $num)
-		local facet=oss$num
-		[[ $(facet_fstype $facet) != zfs ]] ||
-			devname=$(ostdevname $num)
-		mv $devname $tmp/img
-	done
+	mv $OSTDEV1 $tmp/img
 
 	version=$(sed -e 's/\(^[0-9]\+\.[0-9]\+\)\(.*$\)/\1/' $tmp/img/commit |
 			  sed -e 's/\./_/g')	# E.g., "1.8.7" -> "1_8"
@@ -1617,9 +1544,6 @@ t32_test_cleanup() {
 	if $shall_cleanup_ost; then
 		$r $UMOUNT $tmp/mnt/ost || rc=$?
 	fi
-	if $shall_cleanup_ost1; then
-		$r $UMOUNT $tmp/mnt/ost1 || rc=$?
-	fi
 
 	$r rm -rf $tmp
 	rm -rf $tmp
@@ -1779,38 +1703,18 @@ t32_verify_quota() {
 	return 0
 }
 
-getquota() {
-	local spec=$4
-	local uuid=$3
-	local mnt=$5
-
-	sync_all_data > /dev/null 2>&1 || true
-
-	[ "$uuid" != "global" ] || uuid=$mnt
-
-	$LFS quota -v "$1" "$2" $mnt |
-		awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") \
-		{ if (NF == 1) { getline } else { num++ } ; print $num;} }' \
-		| tr -d "*"
-}
-
 t32_test() {
 	local tarball=$1
 	local writeconf=$2
 	local dne_upgrade=${dne_upgrade:-"no"}
 	local dom_upgrade=${dom_upgrade:-"no"}
+	local project_upgrade=${project_upgrade:-"no"}
 	local ff_convert=${ff_convert:-"no"}
-	local pfl_upgrade=${pfl_upgrade:-"no"}
-	local project_quota_upgrade=${project_quota_upgrade:-"no"}
-	local dom_new_upgrade=${dom_new_upgrade:-"no"}
-	local flr_upgrade=${flr_upgrade:-"no"}
 	local shall_cleanup_mdt=false
 	local shall_cleanup_mdt1=false
 	local shall_cleanup_ost=false
-	local shall_cleanup_ost1=false
 	local shall_cleanup_lustre=false
 	local mdt2_is_available=false
-	local ost2_is_available=false
 	local node=$(facet_active_host $SINGLEMDS)
 	local r="do_node $node"
 	local tmp=$TMP/t32
@@ -1831,14 +1735,9 @@ t32_test() {
 	local mdt_dev=$tmp/mdt
 	local mdt2_dev=$tmp/mdt2
 	local ost_dev=$tmp/ost
-	local ost2_dev=$tmp/ost2
 	local stripe_index
 	local stripe_count
 	local dir
-	local pfl_file=$tmp/mnt/lustre/remote_dir/pfl_dir/pfl_file
-	local flr_file=$tmp/mnt/lustre/remote_dir/flr_dir/flr_file
-	local dom_file=$tmp/mnt/lustre/remote_dir/dom_dir/dom_file
-	local quota_dir=$tmp/mnt/lustre/remote_dir/project_quota_dir
 
 	combined_mgs_mds || stop_mgs || error "Unable to stop MGS"
 	trap 'trap - RETURN; t32_test_cleanup' RETURN
@@ -1847,7 +1746,7 @@ t32_test() {
 	nid=$($r $LCTL list_nids | head -1)
 
 	mkdir -p $tmp/mnt/lustre || error "mkdir $tmp/mnt/lustre failed"
-	$r mkdir -p $tmp/mnt/{mdt,mdt1,ost,ost1}
+	$r mkdir -p $tmp/mnt/{mdt,mdt1,ost}
 	$r tar xjvf $tarball -S -C $tmp || {
 		error_noexit "Unpacking the disk image tarball"
 		return 1
@@ -1880,7 +1779,6 @@ t32_test() {
 			ff_convert="no"
 
 	! $r test -f $mdt2_dev || mdt2_is_available=true
-	! $r test -f $ost2_dev || ost2_is_available=true
 
 	if [[ "$mds1_FSTYPE" == zfs ]]; then
 		# import pool first
@@ -1902,12 +1800,21 @@ t32_test() {
 		mdt_dev=t32fs-mdt1/mdt1
 		ost_dev=t32fs-ost1/ost1
 		! $mdt2_is_available || mdt2_dev=t32fs-mdt2/mdt2
-		! $ost2_is_available || ost2_dev=t32fs-ost2/ost2
 		wait_update_facet $SINGLEMDS "$ZPOOL list |
 			awk '/^t32fs-mdt1/ { print \\\$1 }'" "t32fs-mdt1" || {
 				error_noexit "import zfs pool failed"
 				return 1
 			}
+	elif [ "$project_upgrade" != "no" ]; then
+		! $r tune2fs -O project $mdt_dev &&
+			error_noexit "enable project on mdt0 failed" &&
+				return 1
+		$mdt2_is_available && ! $r tune2fs "-O project" $mdt2_dev &&
+			error_noexit "enable project on mdt failed" &&
+				return 1
+		! $r tune2fs -O project $ost_dev &&
+			error_noexit "enable project on mdt failed" &&
+				return 1
 	fi
 
 	$r $LCTL set_param debug="$PTLDEBUG"
@@ -1935,31 +1842,12 @@ t32_test() {
 				error_noexit "Enable mdt quota feature"
 				return 1
 			}
-
-			if [ "$project_quota_upgrade" != "no" ]; then
-				$r $TUNE2FS -O project $mdt_dev || {
-					$r losetup -a
-					error_noexit "tune2fs $mdt_dev failed"
-					return 1
-				}
-				echo "enable project quota on $mdt_dev"
-			fi
-
 			if $mdt2_is_available; then
 				$r $TUNEFS --quota $mdt2_dev || {
 					$r losetup -a
 					error_noexit "Enable mdt quota feature"
 					return 1
 				}
-				if [ "$project_quota_upgrade" != "no" ]; then
-					$r $TUNE2FS -O project $mdt2_dev || {
-						$r losetup -a
-						error_noexit \
-						"tune2fs $mdt2_dev failed"
-						return 1
-					}
-					echo "enable project quota on $mdt2_dev"
-				fi
 			fi
 		fi
 	else
@@ -2003,14 +1891,12 @@ t32_test() {
 			error_noexit "Mounting the MDT"
 			return 1
 		}
-		shall_cleanup_mdt1=true
-		echo "mount new MDT....$mdt2_dev"
 
-		$r $LCTL set_param -n mdt.${fsname}*.enable_remote_dir=1 || {
+		echo "mount new MDT....$mdt2_dev"
+		$r $LCTL set_param -n mdt.${fsname}*.enable_remote_dir=1 ||
 			error_noexit "enable remote dir create failed"
-			return 1
-		}
 
+		shall_cleanup_mdt1=true
 	elif [ "$dne_upgrade" != "no" ]; then
 		local fs2mdsdev=$(mdsdevname 1_2)
 		local fs2mdsvdev=$(mdsvdevname 1_2)
@@ -2020,8 +1906,9 @@ t32_test() {
 			mkfsoptions="--mkfsoptions=\\\"-J size=8\\\""
 		fi
 
-		add $SINGLEMDS $(mkfs_opts mds2 $fs2mdsdev $fsname) --reformat \
-			   $mkfsoptions $fs2mdsdev $fs2mdsvdev > /dev/null || {
+		add $SINGLEMDS $(mds2failover_HOST="" \
+			mkfs_opts mds2 $fs2mdsdev $fsname) --reformat \
+			$mkfsoptions $fs2mdsdev $fs2mdsvdev > /dev/null || {
 			error_noexit "Mkfs new MDT failed"
 			return 1
 		}
@@ -2058,49 +1945,15 @@ t32_test() {
 		error_noexit "tunefs.lustre before mounting the OST"
 		return 1
 	}
-
-	if $ost2_is_available; then
-		$r $TUNEFS --dryrun $ost2_dev || {
-			error_noexit "tunefs.lustre before mounting the OST"
-			return 1
-		}
-	fi
-
 	if [ "$writeconf" ]; then
 		mopts=mgsnode=$nid,$writeconf
-		if [ "$ost1_FSTYPE" == ldiskfs ]; then
+		if [ "$mds1_FSTYPE" == ldiskfs ]; then
 			mopts="loop,$mopts"
 			$r $TUNEFS --quota $ost_dev || {
 				$r losetup -a
 				error_noexit "Enable ost quota feature"
 				return 1
 			}
-
-			if [ "$project_quota_upgrade" != "no" ]; then
-				$r $TUNE2FS -O project $ost_dev || {
-					$r losetup -a
-					error_noexit "tune2fs $ost_dev failed"
-					return 1
-				}
-				echo "enable project quota on $ost_dev"
-			fi
-
-			if $ost2_is_available; then
-				$r $TUNEFS --quota $ost2_dev || {
-					$r losetup -a
-					error_noexit "Enable ost2 quota feature"
-					return 1
-				}
-				if [ "$project_quota_upgrade" != "no" ]; then
-					$r $TUNE2FS -O project $ost2_dev || {
-						$r losetup -a
-						error_noexit \
-						"tune2fs $ost2_dev failed"
-						return 1
-					}
-					echo "enable project quota on $ost2_dev"
-				fi
-			fi
 		fi
 	else
 		mopts=mgsnode=$nid
@@ -2113,15 +1966,6 @@ t32_test() {
 		error_noexit "Mounting the OST"
 		return 1
 	}
-
-	if $ost2_is_available; then
-		$r $MOUNT_CMD -onomgs -o$mopts $ost2_dev $tmp/mnt/ost1 || {
-			error_noexit "Mounting the OST2"
-			return 1
-		}
-		shall_cleanup_ost1=true
-	fi
-
 	shall_cleanup_ost=true
 
 	uuid=$($r $LCTL get_param -n obdfilter.$fsname-OST0000.uuid) || {
@@ -2133,17 +1977,6 @@ t32_test() {
 		return 1
 	fi
 
-	if $ost2_is_available; then
-		uuid=$($r $LCTL get_param -n obdfilter.$fsname-OST0001.uuid) ||{
-			error_noexit "Getting OST1 UUID"
-			return 1
-		}
-		if [ "$uuid" != $fsname-OST0001_UUID ]; then
-			error_noexit "Unexpected OST1 UUID: \"$uuid\""
-			return 1
-		fi
-	fi
-
 	if [[ $PERM_CMD == *"set_param -P"* ]]; then
 		$r $PERM_CMD osc.$fsname-OST0000*.import=connection=$nid || {
 			error_noexit "Setting OST \"failover.node\""
@@ -2267,11 +2100,10 @@ t32_test() {
 		shall_cleanup_lustre=true
 		$r $LCTL set_param debug="$PTLDEBUG"
 
-		# Leave re-enabling this to a separate patch for LU-11558
-		# t32_verify_quota $SINGLEMDS $fsname $tmp/mnt/lustre || {
-		#	error_noexit "verify quota failed"
-		#	return 1
-		#}
+		t32_verify_quota $SINGLEMDS $fsname $tmp/mnt/lustre || {
+			error_noexit "verify quota failed"
+			return 1
+		}
 
 		if $r test -f $tmp/list; then
 			#
@@ -2324,7 +2156,6 @@ t32_test() {
 			echo "list verification skipped"
 		fi
 
-		#non-dom upgrade to dom
 		if [ "$dom_upgrade" != "no" ]; then
 			echo "Check DoM file can be created"
 			$LFS setstripe -E 1M -L mdt -E EOF $tmp/mnt/lustre/dom || {
@@ -2354,40 +2185,12 @@ t32_test() {
 			}
 		fi
 
-		#dom upgrade
-		#$LFS setstripe -E 1M -L mdt -E -1 -S 4M $dom_file
-		if [ "$dom_new_upgrade" != "no" ]; then
-			if ! $mdt2_is_available; then
-				dom_file=$tmp/mnt/lustre/dom_dir/dom_file
-			fi
-			echo "Check DoM file can be accessed"
-			[ $($LFS getstripe -I1 -L $dom_file) == "mdt" ] || {
-				error_noexit "Verify a DoM file"
-				return 1
-			}
-			[ $(stat -c%s $dom_file) == $((2 * 1024 * 1024)) ] || {
-				error_noexit "DoM: bad size after write"
-				return 1
-			}
-		fi
-
-		if [ "$flr_upgrade" != "no" ]; then
-			if ! $mdt2_is_available; then
-				flr_file=$tmp/mnt/lustre/flr_dir/flr_file
-			fi
-			local mirror_count=$($LFS getstripe -N $flr_file)
-			echo "Check FLR file"
-			[ $mirror_count == 2 ] || {
-				error_noexit "FLR mirror count wrong"
-				return 1
-			}
-		fi
-
 		if [ "$dne_upgrade" != "no" ]; then
 			$LFS mkdir -i 1 -c2 $tmp/mnt/lustre/striped_dir || {
 				error_noexit "set striped dir failed"
 				return 1
 			}
+
 			$LFS setdirstripe -D -c2 $tmp/mnt/lustre/striped_dir
 
 			pushd $tmp/mnt/lustre
@@ -2401,69 +2204,6 @@ t32_test() {
 			popd
 		fi
 
-		if [ "$pfl_upgrade" != "no" ]; then
-			local comp_size
-			local comp_cnt
-			local stripe_size
-			local stripe_cnt
-			local comp_id
-
-			echo "Check PFL file"
-			if ! $mdt2_is_available; then
-				pfl_file=$tmp/mnt/lustre/pfl_dir/pfl_file
-			fi
-			comp_cnt=$($LFS getstripe --component-count $pfl_file)
-			[ $comp_cnt == 2 ] || {
-				error_noexit "wrong comp_cnt $comp_cnt"
-				return 1
-			}
-
-			comp_size=$($LFS getstripe -I1 -E $pfl_file)
-			[ $comp_size == 2097152 ] || {
-				error_noexit "wrong component size $comp_size"
-				return 1
-			}
-
-			comp_id=$($LFS getstripe -I1 -i $pfl_file)
-			[ $comp_id == 0 ] || {
-				error_noexit "wrong comp id $comp_id"
-				return 1
-			}
-
-			comp_id=$($LFS getstripe -I2 -i $pfl_file)
-			[ $comp_id -eq 1 ] || {
-				error_noexit "wrong comp id $comp_id"
-				return 1
-			}
-
-			stripe_size=$($LFS getstripe -I1 -S $pfl_file)
-			[ $stripe_size -eq 1048576 ] || {
-				error_noexit "wrong stripe size $stripe_size"
-				return 1
-			}
-
-			stripe_size=$($LFS getstripe -I2 -S $pfl_file)
-			[ $comp_size -eq 2097152 ] || {
-				error_noexit "wrong component size $comp_size"
-				return 1
-			}
-		fi
-
-		if [ "$project_quota_upgrade" != "no" ]; then
-			if ! $mdt2_is_available; then
-				quota_dir=$tmp/mnt/lustre/project_quota_dir
-			fi
-			local hardlimit
-
-			echo "Check Project Quota"
-			hardlimit=$(getquota -p $T32_PRJID global 3 \
-							${tmp}/mnt/lustre)
-			[ $hardlimit == $T32_PROLIMIT ] || {
-				error_noexit "wrong hardlimit $hardlimit"
-				return 1
-			}
-		fi
-
 		# If it is upgrade from DNE (2.5), then rename the remote dir,
 		# which is created in 2.5 to striped dir.
 		if $mdt2_is_available && [[ "$dne_upgrade" != "no" ]]; then
@@ -2517,11 +2257,6 @@ t32_test() {
 			$r cat $tmp/sha1sums | sort -k 2 >$tmp/sha1sums.orig
 			if [ "$dne_upgrade" != "no" ]; then
 				pushd $tmp/mnt/lustre/striped_dir
-			elif [ "$pfl_upgrade" != "no" ] ||
-				[ "$flr_upgrade" != "no" ] ||
-				[ "$dom_new_upgrade" != "no" ] ||
-				[ "$project_quota_upgrade" != "no" ]; then
-				pushd $tmp/mnt/lustre/remote_dir
 			else
 				pushd $tmp/mnt/lustre
 			fi
@@ -2572,18 +2307,6 @@ t32_test() {
 			echo "sha1sum verification skipped"
 		fi
 
-		# PFL write test after sha1sum check
-		if [ "$pfl_upgrade" != "no" ]; then
-			local rw_len=$((3 * 1034 * 1024))
-			if ! $mdt2_is_available; then
-				pfl_file=$tmp/mnt/lustre/pfl_dir/pfl_file
-			fi
-			small_write $pfl_file $rw_len || {
-				error_noexit "PFL RW Failed"
-				return 1
-			}
-		fi
-
 		if [ "$dne_upgrade" != "no" ]; then
 			rm -rf $tmp/mnt/lustre/striped_dir || {
 				error_noexit "remove remote dir failed"
@@ -2657,6 +2380,13 @@ t32_test() {
 			return 1
 		}
 
+		if [ "$mds1_FSTYPE" == ldiskfs -a \
+		    "$project_upgrade" != "no" ]; then
+			! $LFS project -d -p 1 $tmp/mnt/lustre/* &&
+				error_noexit "set project failed" &&
+					return 1
+		fi
+
 		[[ $(do_facet mds1 pgrep orph_.*-MDD | wc -l) == 0 ]] ||
 			error "MDD orphan cleanup thread not quit"
 
@@ -2689,19 +2419,8 @@ t32_test() {
 			error_noexit "Unmounting the OST"
 			return 1
 		}
-		if $ost2_is_available; then
-			$r $UMOUNT $tmp/mnt/ost1 || {
-				error_noexit "Unmounting the OST1"
-				return 1
-			}
-			shall_cleanup_ost1=false
-		fi
-
-		if [[ $ost1_FSTYPE == zfs ]]; then
-			$r "$ZPOOL export t32fs-ost1"
-			if $ost2_is_available; then
-				$r "$ZPOOL export t32fs-ost2"
-			fi
+		if [[ "$mds1_FSTYPE" == zfs ]]; then
+		    $r "$ZPOOL export t32fs-ost1"
 		fi
 		shall_cleanup_ost=false
 
@@ -2794,12 +2513,17 @@ test_32d() {
 
 	t32_check
 	for tarball in $tarballs; do
-		banner "testing $tarball upgrade with ff convert"
-		ff_convert=yes t32_test $tarball || rc=$?
+		banner "testing $tarball upgrade with ff convert and project upgrade"
+		project_upgrade="no"
+		[[ "$MDS1_VERSION" -ge $(version_code 2.13.54) ]] &&
+			[[ "$tarball" =~ "disk2_4-ldiskfs" ]] &&
+				project_upgrade="yes"
+		project_upgrade=$project_upgrade ff_convert=yes t32_test \
+			$tarball || rc=$?
 	done
 	return $rc
 }
-run_test 32d "convert ff test"
+run_test 32d "convert ff and project quota upgrade test"
 
 test_32e() {
 	[[ "$MDS1_VERSION" -ge $(version_code 2.10.56) ]] ||
@@ -2820,42 +2544,6 @@ test_32e() {
 }
 run_test 32e "dom upgrade test"
 
-test_32f() {
-	[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
-		skip "Need MDS version at least 2.10.56"
-
-	local tarballs
-	local tarball
-	local rc=0
-
-	t32_check
-	for tarball in $tarballs; do
-		echo $tarball | grep "2_10" || continue
-		pfl_upgrade=yes project_quota_upgrade=yes \
-		t32_test $tarball writeconf || let "rc += $?"
-	done
-	return $rc
-}
-run_test 32f "pfl upgrade test"
-
-test_32g() {
-	[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
-		skip "Need MDS version at least 2.10.56"
-
-	local tarballs
-	local tarball
-	local rc=0
-
-	t32_check
-	for tarball in $tarballs; do
-		echo $tarball | grep "2_12" || continue
-		flr_upgrade=yes dom_new_upgrade=yes \
-		t32_test $tarball writeconf || let "rc += $?"
-	done
-	return $rc
-}
-run_test 32g "flr/dom upgrade test"
-
 test_33a() { # bug 12333, was test_33
 	local FSNAME2=test-123
 	local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
@@ -3600,7 +3288,7 @@ test_43a() {
 	echo "222" > $DIR/$tfile-rootfile || error "write 2 failed"
 	chmod go-rw $DIR/$tfile-rootfile  || error "chmod 2 faield"
 
-	mkdir $DIR/$tdir-rootdir || error "mkdir failed"
+	mkdir_on_mdt0 $DIR/$tdir-rootdir || error "mkdir failed"
 	chmod go-rwx $DIR/$tdir-rootdir || error "chmod 3 failed"
 	touch $DIR/$tdir-rootdir/tfile-1 || error "touch failed"
 
@@ -4268,7 +3956,7 @@ test_50i() {
 	start_ost2 || error "Unable to start OST2"
 	mount_client $MOUNT || error "client start failed"
 
-	mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
 
 	if [[ $PERM_CMD == *"set_param -P"* ]]; then
 		$PERM_CMD mdc.${FSNAME}-MDT0001-mdc-*.active=0 &&
@@ -5042,6 +4730,10 @@ test_63() {
 		skip "ldiskfs module has not been loaded"
 	fi
 
+	if grep -q "CONFIG_DEBUG_LOCK_ALLOC=y" /boot/config-$(uname -r); then
+		skip "test is not compatible with CONFIG_DEBUG_LOCK_ALLOC=y"
+	fi
+
 	echo "$inode_slab ldiskfs inodes per page"
 	[ "${inode_slab%.*}" -ge "3" ] && return 0
 
@@ -5343,8 +5035,9 @@ test_69() {
 	if [ $num_create -gt 0 ]; then
 		# Check the number of inodes available on OST0
 		local files=0
-		local ifree=$($LFS df -i $MOUNT | awk '/OST0000/ { print $4 }')
-		log "On OST0, $ifree inodes available. Want $num_create."
+		local ifree=$($LFS df -i $MOUNT |
+			awk '/OST0000/ { print $4 }'; exit ${PIPESTATUS[0]})
+		log "On OST0, $ifree inodes available. Want $num_create. rc=$?"
 
 		$LFS setstripe -i 0 $DIR/$tdir ||
 			error "$LFS setstripe -i 0 $DIR/$tdir failed"
@@ -5380,8 +5073,9 @@ test_69() {
 	local idx=$($LFS getstripe -i $DIR/$tdir/$tfile-last)
 	[ $idx -ne 0 ] && error "$DIR/$tdir/$tfile-last on $idx not 0" || true
 
-	local iused=$($LFS df -i $MOUNT | awk '/OST0000/ { print $3 }')
-	log "On OST0, $iused used inodes"
+	local iused=$($LFS df -i $MOUNT |
+		awk '/OST0000/ { print $3 }'; exit ${PIPESTATUS[0]})
+	log "On OST0, $iused used inodes rc=$?"
 	[ $iused -ge $((ost_max_pre/2 + 1000)) ] &&
 		error "OST replacement created too many inodes; $iused"
 	cleanup || error "cleanup failed with $?"
@@ -6328,7 +6022,8 @@ test_82a() { # LU-4665
 	mount_client $MOUNT || error "mount client $MOUNT failed"
 	wait_osts_up
 
-	$LFS df $MOUNT || error "$LFS df $MOUNT failed"
+	$LFS df $MOUNT
+	check_lfs_df_ret_val $? || error "$LFS df $MOUNT failed"
 	mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
 
 	stack_trap "do_nodes $(comma_list $(mdts_nodes)) \
@@ -6443,7 +6138,8 @@ test_82b() { # LU-4665
 	mount_client $MOUNT || error "mount client $MOUNT failed"
 
 	wait_osts_up
-	$LFS df $MOUNT || error "$LFS df $MOUNT failed"
+	$LFS df $MOUNT
+	check_lfs_df_ret_val $? || error "$LFS df $MOUNT failed"
 	mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
 
 	# Create a new pool and add OSTs into it.
@@ -6641,23 +6337,15 @@ test_86() {
 	[[ "$MDS1_VERSION" -ge $(version_code 2.7.56) ]] ||
 		skip "Need server version newer than 2.7.55"
 
-	local OST_OPTS="$(mkfs_opts ost1 $(ostdevname 1)) \
-		--reformat $(ostdevname 1) $(ostvdevname 1)"
-
 	local NEWSIZE=1024
 	local OLDSIZE=$(do_facet ost1 "$DEBUGFS -c -R stats $(ostdevname 1)" |
 		awk '/Flex block group size: / { print $NF; exit; }')
 
-	local opts=OST_OPTS
-	if [[ ${!opts} != *mkfsoptions* ]]; then
-		eval opts=\"${!opts} \
-			--mkfsoptions='\\\"-O flex_bg -G $NEWSIZE\\\"'\"
-	else
-		val=${!opts//--mkfsoptions=\\\"/ \
-			--mkfsoptions=\\\"-O flex_bg -G $NEWSIZE }
-		eval opts='${val}'
-	fi
+	[ "$OLDSIZE" == "$NEWSIZE" ] && skip "$NEWSIZE groups already"
 
+	local opts=" -O flex_bg -G $NEWSIZE"
+	opts=$(OST_FS_MKFS_OPTS+="$opts" mkfs_opts ost1 $(ostdevname 1))
+	opts+=" --reformat $(ostdevname 1) $(ostvdevname 1)"
 	echo "params: $opts"
 
 	trap cleanup_86 EXIT ERR
@@ -6804,12 +6492,15 @@ test_89() { # LU-7131
 
 	stopall
 
-	[ "$mds1_FSTYPE" == zfs ] && import_zpool mds1
+	if [[ "$mds1_FSTYPE" == zfs ]]; then
+		import_zpool mds1 || return ${PIPESTATUS[0]}
+	fi
+
 	# Check that parameters are added correctly
 	echo "tunefs --param $key=$val1"
-	do_facet mds "$TUNEFS --param $key=$val1 $mdsdev >/dev/null" ||
+	do_facet mds1 "$TUNEFS --param $key=$val1 $mdsdev >/dev/null" ||
 		error "tunefs --param $key=$val1 failed"
-	params=$(do_facet mds $TUNEFS --dryrun $mdsdev) ||
+	params=$(do_facet mds1 $TUNEFS --dryrun $mdsdev) ||
 		error "tunefs --dryrun failed"
 	params=${params##*Parameters:}
 	params=${params%%exiting*}
@@ -6818,9 +6509,9 @@ test_89() { # LU-7131
 
 	# Check that parameters replace existing instances when added
 	echo "tunefs --param $key=$val2"
-	do_facet mds "$TUNEFS --param $key=$val2 $mdsdev >/dev/null" ||
+	do_facet mds1 "$TUNEFS --param $key=$val2 $mdsdev >/dev/null" ||
 		error "tunefs --param $key=$val2 failed"
-	params=$(do_facet mds $TUNEFS --dryrun $mdsdev) ||
+	params=$(do_facet mds1 $TUNEFS --dryrun $mdsdev) ||
 		error "tunefs --dryrun failed"
 	params=${params##*Parameters:}
 	params=${params%%exiting*}
@@ -6831,9 +6522,9 @@ test_89() { # LU-7131
 
 	# Check that a parameter is erased properly
 	echo "tunefs --erase-param $key"
-	do_facet mds "$TUNEFS --erase-param $key $mdsdev >/dev/null" ||
+	do_facet mds1 "$TUNEFS --erase-param $key $mdsdev >/dev/null" ||
 		error "tunefs --erase-param $key failed"
-	params=$(do_facet mds $TUNEFS --dryrun $mdsdev) ||
+	params=$(do_facet mds1 $TUNEFS --dryrun $mdsdev) ||
 		error "tunefs --dryrun failed"
 	params=${params##*Parameters:}
 	params=${params%%exiting*}
@@ -6841,22 +6532,25 @@ test_89() { # LU-7131
 		error "on-disk parameter not erased correctly via tunefs"
 
 	# Check that all the parameters are erased
+	do_facet mds1 "$TUNEFS --param $key=$val1 $mdsdev >/dev/null" ||
+		error "tunefs --param $key=$val1 failed"
 	echo "tunefs --erase-params"
-	do_facet mds "$TUNEFS --erase-params $mdsdev >/dev/null" ||
+	do_facet mds1 "$TUNEFS --erase-params $mdsdev >/dev/null" ||
 		error "tunefs --erase-params failed"
-	params=$(do_facet mds $TUNEFS --dryrun $mdsdev) ||
+	params=$(do_facet mds1 $TUNEFS --dryrun $mdsdev) ||
 		error "tunefs --dryrun failed"
 	params=${params##*Parameters:}
 	params=${params%%exiting*}
-	[ -z $params ] ||
-		error "all on-disk parameters not erased correctly via tunefs"
+	params=$(echo $params | tr ' ' '\n')
+	[ -z "$params" ] ||
+		error "all on-disk parameters not erased correctly via tunefs $params"
 
 	# Check the order of options --erase-params and --param
 	echo "tunefs --param $key=$val1 --erase-params"
-	do_facet mds \
+	do_facet mds1 \
 		"$TUNEFS --param $key=$val1 --erase-params $mdsdev >/dev/null"||
 		error "tunefs --param $key=$val1 --erase-params failed"
-	params=$(do_facet mds $TUNEFS --dryrun $mdsdev) ||
+	params=$(do_facet mds1 $TUNEFS --dryrun $mdsdev) ||
 		error "tunefs --dryrun failed"
 	params=${params##*Parameters:}
 	params=${params%%exiting*}
@@ -7022,10 +6716,41 @@ test_90b() {
 }
 run_test 90b "check max_mod_rpcs_in_flight is enforced after update"
 
+save_params_90c() {
+	# get max_rpcs_in_flight value
+	mrif_90c=$($LCTL get_param -n \
+		   mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
+	echo "max_rpcs_in_flight is $mrif_90c"
+
+	# get max_mod_rpcs_in_flight value
+	mmrif_90c=$($LCTL get_param -n \
+		    mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight)
+	echo "max_mod_rpcs_in_flight is $mmrif_90c"
+
+	# get MDT max_mod_rpcs_per_client value
+	mmrpc_90c=$(do_facet mds1 \
+		    cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
+	echo "max_mod_rpcs_per_client is $mmrpc_90c"
+}
+
+restore_params_90c() {
+	trap 0
+
+	# restore max_rpcs_in_flight value
+	do_facet mgs $LCTL set_param -P \
+		mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c
+
+	# restore max_mod_rpcs_in_flight value
+	do_facet mgs $LCTL set_param -P \
+		mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mmrif_90c
+
+	# restore MDT max_mod_rpcs_per_client value
+	do_facet mds1 "echo $mmrpc_90c > \
+		       /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+}
+
 test_90c() {
 	local tmp
-	local mrif
-	local mmrpc
 
 	setup
 
@@ -7041,42 +6766,64 @@ test_90c() {
 		skip "Client not able to send multiple modify RPCs in parallel"
 	fi
 
-	# get max_rpcs_in_flight value
-	mrif=$($LCTL get_param -n mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
-	echo "max_rpcs_in_flight is $mrif"
-
-	# get MDT max_mod_rpcs_per_client
-	mmrpc=$(do_facet mds1 \
-		    cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
-	echo "max_mod_rpcs_per_client is $mmrpc"
+	save_params_90c
+	stack_trap restore_params_90c
 
 	# testcase 1
 	# attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value
 	# prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value
-	umount_client $MOUNT
-	do_facet mds1 \
-		"echo $mrif > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
-	mount_client $MOUNT
+	do_facet mds1 "echo $mrif_90c > \
+		       /sys/module/mdt/parameters/max_mod_rpcs_per_client"
 
-	$LCTL set_param \
-	    mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif &&
-	    error "set max_mod_rpcs_in_flight to $mrif should fail"
+	# if max_mod_rpcs_in_flight is set to be equal to or larger than
+	# max_rpcs_in_flight, then max_rpcs_in_flight will be increased
+	if [[ "$CLIENT_VERSION" -ge $(version_code 2.13.53) ]]; then
+		$LCTL set_param \
+		mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c ||
+			error "set max_mod_rpcs_in_flight to $mrif_90c failed"
+
+		local new_mrif=$($LCTL get_param -n \
+				 mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
+		((new_mrif == mrif_90c + 1)) ||
+			error "max_rpcs_in_flight was not increased"
+	fi
 
 	umount_client $MOUNT
-	do_facet mds1 \
-		"echo $mmrpc > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+	do_facet mds1 "echo $mmrpc_90c > \
+		       /sys/module/mdt/parameters/max_mod_rpcs_per_client"
 	mount_client $MOUNT
 
 	# testcase 2
 	# attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1
 	# prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2
 	$LCTL set_param \
-	    mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc + 2))
+		mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc_90c + 2))
 
 	$LCTL set_param \
-	    mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc + 1)) &&
-	    error "set max_mod_rpcs_in_flight to $((mmrpc + 1)) should fail"
+	mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc_90c + 1)) &&
+	error "set max_mod_rpcs_in_flight to $((mmrpc_90c + 1)) should fail"
+
+	# testcase 3
+	# attempt to set max_mod_rpcs_in_flight permanently
+	do_facet mgs $LCTL set_param -P \
+		mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c
+
+	do_facet mgs $LCTL set_param -P \
+		mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c
+
+	remount_client $MOUNT
 
+	wait_update_facet --verbose client "$LCTL get_param -n \
+		mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight" \
+		"$((mrif_90c + 1))" ||
+		error "expected '$((mrif_90c + 1))' for max_rpcs_in_flight"
+
+	wait_update_facet --verbose client "$LCTL get_param -n \
+		mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight" \
+		"$mrif_90c" ||
+		error "expected '$mrif_90c' for max_mod_rpcs_in_flight"
+
+	restore_params_90c
 	cleanup
 }
 run_test 90c "check max_mod_rpcs_in_flight update limits"
@@ -7156,7 +6903,7 @@ check_uuid_on_ost() {
 
 check_uuid_on_mdt() {
 	local nid=$1
-	do_facet $SINGLEMDS "$LCTL get_param mdt.${FSNAME}*.exports.'$nid'.uuid"
+	do_facet $SINGLEMDS "$LCTL get_param mdt.${mds1_svc}*.exports.'$nid'.uuid"
 }
 
 test_91() {
@@ -7200,6 +6947,7 @@ test_91() {
 	echo "evict $nid"
 	do_facet $SINGLEMDS \
 		"$LCTL set_param -n mdt.${mds1_svc}.evict_client nid:$nid"
+	sleep 1 # eviction above is async, give it some time to proceed
 
 	found=$(check_uuid_on_mdt $nid | grep $uuid)
 	[ -n "$found" ] && error "found $uuid $nid on MDT"
@@ -7222,63 +6970,33 @@ generate_ldev_conf() {
 	local ldevconfpath=$1
 	local fstype=
 	local fsldevformat=""
-	touch $ldevconfpath
+	rm -f $ldevconfpath
 
-	fstype=$(facet_fstype mgs)
-	if [ "$fstype" == zfs ]; then
-		fsldevformat="$fstype:"
-	else
+	local facets="mgs,$(get_facets OST),$(get_facets MDS)"
+	for facet in ${facets//,/ }; do
 		fsldevformat=""
-	fi
-
-	printf "%s\t-\t%s-MGS0000\t%s%s\n" \
-		$mgs_HOST \
-		$FSNAME \
-		$fsldevformat \
-		$(mgsdevname) > $ldevconfpath
-
-	local mdsfo_host=$mdsfailover_HOST;
-	if [ -z "$mdsfo_host" ]; then
-		mdsfo_host="-"
-	fi
-
-	for num in $(seq $MDSCOUNT); do
-		fstype=$(facet_fstype mds$num)
-		if [ "$fstype" == zfs ]; then
-			fsldevformat="$fstype:"
-		else
-			fsldevformat=""
-		fi
-
-		printf "%s\t%s\t%s-MDT%04d\t%s%s\n" \
-			$mds_HOST \
-			$mdsfo_host \
-			$FSNAME \
-			$num \
-			$fsldevformat \
-			$(mdsdevname $num) >> $ldevconfpath
-	done
-
-	local ostfo_host=$ostfailover_HOST;
-	if [ -z "$ostfo_host" ]; then
-		ostfo_host="-"
-	fi
-
-	for num in $(seq $OSTCOUNT); do
-		fstype=$(facet_fstype ost$num)
-		if [ "$fstype" == zfs ]; then
-			fsldevformat="$fstype:"
-		else
-			fsldevformat=""
+		fstype=$(facet_fstype $facet)
+		[ "$fstype" = zfs ] && fsldevformat="$fstype:"
+		local host=$(facet_host $facet)
+		local fo="-"
+		local varfo=${facet}failover_HOST
+		if [ $facet == mgs ] && combined_mgs_mds; then
+			varfo=mds1failover_HOST
 		fi
-
-		printf "%s\t%s\t%s-OST%04d\t%s%s\n" \
-			$ost_HOST \
-			$ostfo_host \
+		[ -n "${!varfo}" ] && fo=${!varfo}
+		local type=$(echo $facet | tr -d "[:digit:]" | \
+			tr "[:lower:]" "[:upper:]" | sed s/MDS/MDT/ )
+		local num=1
+		[ ${facet} == mgs ] ||
+			num=$(facet_number $facet)
+		printf "%s\t%s\t%s-%s%04x\t%s%s\n" \
+			${host} \
+			${fo} \
 			$FSNAME \
-			$num \
+			$type \
+			$(( num - 1 )) \
 			$fsldevformat \
-			$(ostdevname $num) >> $ldevconfpath
+			$(facet_device $facet) >> $ldevconfpath
 	done
 
 	echo "----- $ldevconfpath -----"
@@ -7300,8 +7018,8 @@ generate_nids() {
 }
 
 compare_ldev_output() {
-	ldev_output=$1
-	expected_output=$2
+	local ldev_output=$1
+	local expected_output=$2
 
 	sort $expected_output -o $expected_output
 	sort $ldev_output -o $ldev_output
@@ -7332,30 +7050,27 @@ test_92() {
 	generate_nids $NIDSPATH
 
 	# echo the mgs nid and compare it to environment variable MGSNID
-	# also, ldev.conf and nids is a server side thing, use the OSS
-	# hostname
-	local output
-	output=$($LDEV -c $LDEVCONFPATH -H $ost_HOST -n $NIDSPATH echo %m)
-
-	echo "-- START OF LDEV OUTPUT --"
-	echo -e "$output"
-	echo "--- END OF LDEV OUTPUT ---"
-
-	# ldev failed, error
-	if [ $? -ne 0 ]; then
-		rm $LDEVCONFPATH $NIDSPATH
-		error "ldev failed to execute!"
-	fi
-
-	# need to process multiple lines because of combined MGS and MDS
-	echo -e $output | awk '{ print $2 }' | while read -r line ; do
-		if [ "$line" != "$MGSNID" ]; then
-			rm $LDEVCONFPATH $NIDSPATH
-			error "ldev failed mgs nid '$line', expected '$MGSNID'"
-		fi
+	local facets="$(get_facets OST),$(get_facets MDS),mgs"
+	for facet in ${facets//,/ }; do
+		local host=$(facet_host $facet)
+		local output=$($LDEV -c $LDEVCONFPATH -H $host -n $NIDSPATH echo %m)
+
+		echo "-- START OF LDEV OUTPUT --"
+		echo -e "$output"
+		echo "--- END OF LDEV OUTPUT ---"
+
+		[ -z "$output" ] &&
+			error "ldev failed to execute!"
+
+		# need to process multiple lines because of
+		# several targets on host
+		echo -e $output | awk '{ print $2 }' | while read -r line ; do
+			[ "$line" = "$MGSNID" ] ||
+				error "ldev failed mgs nid '$line', \
+					expected '$MGSNID'"
+		done
 	done
-
-	rm $LDEVCONFPATH $NIDSPATH
+	rm -f $LDEVCONFPATH $NIDSPATH
 }
 run_test 92 "ldev returns MGS NID correctly in command substitution"
 
@@ -7414,11 +7129,11 @@ test_94() {
 	printf "%s-MGS0000\n" $FSNAME > $EXPECTED_OUTPUT
 
 	for num in $(seq $MDSCOUNT); do
-		printf "%s-MDT%04d\n" $FSNAME $num >> $EXPECTED_OUTPUT
+		printf "%s-MDT%04x\n" $FSNAME $((num - 1)) >> $EXPECTED_OUTPUT
 	done
 
 	for num in $(seq $OSTCOUNT); do
-		printf "%s-OST%04d\n" $FSNAME $num >> $EXPECTED_OUTPUT
+		printf "%s-OST%04x\n" $FSNAME $((num - 1)) >> $EXPECTED_OUTPUT
 	done
 
 	compare_ldev_output $LDEV_OUTPUT $EXPECTED_OUTPUT
@@ -7546,19 +7261,13 @@ test_96() {
 
 	echo "$mgs_HOST-$(facet_fstype mgs)" > $EXPECTED_OUTPUT
 
-	if [ "$mgs_HOST" == "$mds_HOST" ]; then
-		for num in $(seq $MDSCOUNT); do
-			echo "$mds_HOST-$(facet_fstype mds$num)" \
+	local facets="$(get_facets OST),$(get_facets MDS)"
+	for facet in ${facets//,/ }; do
+		local host=$(facet_host $facet)
+		[ "$mgs_HOST" == "$host" ] &&
+			echo "$host-$(facet_fstype $facet)" \
 			>> $EXPECTED_OUTPUT
-		done
-	fi
-
-	if [ "$mgs_HOST" == "$ost_HOST" ]; then
-		for num in $(seq $OSTCOUNT); do
-			echo "$ost_HOST-$(facet_fstype ost$num)" \
-			>> $EXPECTED_OUTPUT
-		done
-	fi
+	done
 
 	compare_ldev_output $LDEV_OUTPUT $EXPECTED_OUTPUT
 
@@ -7594,7 +7303,7 @@ test_97() {
 	fi
 
 	for num in $(seq $MDSCOUNT); do
-		printf "%s-MDT%04d\n" $FSNAME $num >> $EXPECTED_OUTPUT
+		printf "%s-MDT%04x\n" $FSNAME $((num - 1)) >> $EXPECTED_OUTPUT
 	done
 
 	compare_ldev_output $LDEV_OUTPUT $EXPECTED_OUTPUT
@@ -7614,7 +7323,7 @@ test_97() {
 
 	rm $EXPECTED_OUTPUT
 	for num in $(seq $OSTCOUNT); do
-		printf "%s-OST%04d\n" $FSNAME $num >> $EXPECTED_OUTPUT
+		printf "%s-OST%04x\n" $FSNAME $((num - 1)) >> $EXPECTED_OUTPUT
 	done
 
 	compare_ldev_output $LDEV_OUTPUT $EXPECTED_OUTPUT
@@ -7722,7 +7431,7 @@ test_100() {
 }
 run_test 100 "check lshowmount lists MGS, MDT, OST and 0@lo"
 
-test_101() {
+test_101a() {
 	local createmany_pid
 	local dev=$FSNAME-OST0000-osc-MDT0000
 	setup
@@ -7749,7 +7458,29 @@ test_101() {
 	unlinkmany $DIR1/$tdir/$tfile-%d 50000
 	cleanup
 }
-run_test 101 "Race MDT->OST reconnection with create"
+run_test 101a "Race MDT->OST reconnection with create"
+
+test_101b () {
+	local dev=$FSNAME-OST0000-osc-MDT0000
+	local dir=$DIR1/$tdir
+	setup
+
+	mkdir $dir
+	$LFS setstripe -c 1 -i 0 $dir
+	do_facet $SINGLEMDS "$LCTL --device $dev deactivate;"
+#define OBD_FAIL_OSP_CON_EVENT_DELAY 0x2107
+	do_facet mds1 "$LCTL set_param fail_loc=0x80002107 fail_val=20"
+	do_facet $SINGLEMDS "$LCTL --device $dev activate;"
+	stop_ost
+	sleep 25
+	start_ost
+
+	wait_osc_import_state client ost1 FULL
+	touch $dir/$tfile || error "Can't create file"
+
+	cleanup
+}
+run_test 101b "Race events DISCONNECT and ACTIVE in osp"
 
 test_102() {
 	[[ "$MDS1_VERSION" -gt $(version_code 2.9.53) ]] ||
@@ -8288,7 +8019,6 @@ test_108b() {
 }
 run_test 108b "migrate from ZFS to ldiskfs"
 
-
 #
 # set number of permanent parameters
 #
@@ -8713,6 +8443,8 @@ test_112() {
 	(( $val == 1 )) || error "obdfilter.$FSNAME-OST0001*.no_precreate=$val"
 
 	mount_client $MOUNT || error "mount client failed"
+	wait_osc_import_state mds1 ost1 FULL
+	wait_osc_import_state client ost1 FULL
 	wait_osc_import_state client ost2 FULL
 
 	$LFS setstripe -i 0 $DIR/$tfile.0 ||
@@ -8724,6 +8456,8 @@ test_112() {
 	sleep_maxage
 	$LFS setstripe -i 1 $DIR/$tfile.2 ||
 		error "failed to create $tfile.2 on ost1 facet"
+	# files not cleaned with ONLY_REPEAT because of client unmount below
+	rm -f $DIR/$tfile.[012]
 	stop_ost2 || error "stop ost2 facet failed"
 	cleanup
 }
@@ -8748,20 +8482,21 @@ test_115() {
 	echo "$dbfs_ver" | egrep -w "1.44.3.wc1|1.44.5.wc1|1.45.2.wc1" &&
 		skip_env "This version of debugfs doesn't show inode number"
 
-	is_dm_flakey_dev $SINGLEMDS $(mdsdevname 1) &&
-		skip "This test can not be executed on flakey dev"
-
 	IMAGESIZE=$((3072 << 30)) # 3072 GiB
 
 	stopall
 
+	local saved_flakey=${FLAKEY}
+	stack_trap "FLAKEY=$saved_flakey" EXIT
+	FLAKEY=false
+
 	echo "client1: "
 	lctl dl
 	mount | grep lustre
 	echo "mds1: "
-	do_facet mds1 "hostname; ifconfig; lctl dl; mount"
+	do_facet mds1 "hostname; lctl dl; mount"
 	echo "ost1: "
-	do_facet ost1 "hostname; ifconfig; lctl dl; mount"
+	do_facet ost1 "hostname; lctl dl; mount"
 	# We need MDT size 3072GB, because it is smallest
 	# partition that can store 2B inodes
 	do_facet $SINGLEMDS "mkdir -p $TMP/$tdir"
@@ -8788,21 +8523,23 @@ test_115() {
 
 	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir fail"
 	goal="/sys/fs/ldiskfs/$(basename $mdsdev)/inode_goal"
-echo goal: $goal
+	echo goal: $goal
 	# 2147483648 is 0x80000000
 	do_facet $SINGLEMDS "echo 2147483648 >> $goal; grep . $goal"
 	touch $DIR/$tdir/$tfile
 
-	# attrs from 1 to 15 go to block, 16th - to inode
-	for i in {1..16}; do
+	# attrs from 1 to 16 go to block, 17th - to inode
+	for i in {1..17}; do
 		local nm="trusted.ea$i"
 		setfattr -n $nm -v $(printf "xattr%0250d" $i) $DIR/$tdir/$tfile
 	done
 
+	do_facet $SINGLEMDS $DEBUGFS -c -R "stat ROOT/$tdir/$tfile" $mdsdev
+
 	# inode <2147483649> trusted.ea16 (255)
 	local inode_num=$(do_facet $SINGLEMDS \
 			"$DEBUGFS -c -R 'stat ROOT/$tdir/$tfile' $mdsdev" |
-			 awk '/ea16/ { print $2 }' |
+			 awk '/ea17/ { print $2 }' |
 			 sed -e 's/>//' -e 's/<//' -e 's/\"//')
 	echo "inode num: $inode_num"
 	[ $inode_num -ge 2147483648 ] || error "inode $inode_num too small"
@@ -8844,7 +8581,10 @@ test_116() {
 
 	MDSSIZE=$old_mdssize
 	MDSISIZE=$old_mdsisize
-	do_facet $SINGLEMDS $MKFS $opts17t $tmpmnt/$mdtimg ||
+
+	do_facet $SINGLEMDS $MKFS \
+		${opts17t/-E /-E lazy_itable_init,lazy_journal_init,} \
+		$tmpmnt/$mdtimg ||
 		error "failed to mkfs for $tmpmnt/$mdtimg"
 
 	do_facet $SINGLEMDS $TUNE2FS -l $tmpmnt/$mdtimg |
@@ -8862,6 +8602,50 @@ test_117() {
 }
 run_test 117 "lctl get_param return errors properly"
 
+test_119() {
+	local had_config
+	local size_mb
+
+	[[ "$MDSCOUNT" -ge 2 ]] || skip "Need more at least 2 MDTs"
+
+	had_config=$(do_facet mds1 "$LCTL get_param debug | grep config")
+	do_facet mds1 "$LCTL set_param debug=+config"
+	do_facet mds1 "$LCTL clear"
+
+	setup
+	do_facet mds2 "$TUNEFS --writeconf $(mdsdevname 2)" &>/dev/null
+	# mount after writeconf will make "add osp" added to mdt0 config:
+	# 53 (224)marker  60 (flags=0x01, v2.5.1.0) lustre-MDT0001  'add osp'
+	# 54 (080)add_uuid  nid=...  0:  1:...
+	# 55 (144)attach    0:lustre-MDT0001-osp-MDT0000  1:osp  2:...
+	# 56 (144)setup     0:lustre-MDT0001-osp-MDT0000  1:...  2:...
+	# 57 (136)modify_mdc_tgts add 0:lustre-MDT0000-mdtlov  1:...  2:1  3:1
+	# duplicate modify_mdc_tgts caused crashes
+
+	debug_size_save
+	# using larger debug_mb size to avoid lctl dk log truncation
+	size_mb=$((DEBUG_SIZE_SAVED * 4))
+	for i in {1..3}; do
+		stop_mdt 2
+		# though config processing stops after failed attach and setup
+		# it will proceed after the failed command after each writeconf
+		# this is the original scenario of the issue
+		do_facet mds2 "$TUNEFS --writeconf $(mdsdevname 2)" &>/dev/null
+		do_facet mds1 "$LCTL set_param debug_mb=$size_mb"
+		start_mdt 2
+
+		wait_update_facet_cond mds1 \
+			"$LCTL dk | grep -c Processed.log.$FSNAME-MDT0000" \
+			">" 1 300
+	done
+	debug_size_restore
+
+	[[ -z "$had_config" ]] && do_facet mds1 lctl set_param debug=-config
+
+	reformat
+}
+run_test 119 "writeconf on slave mdt shouldn't duplicate mdc/osp and crash"
+
 test_120() { # LU-11130
 	[ "$MDSCOUNT" -lt 2 ] && skip "mdt count < 2"
 	[ "$mds1_FSTYPE" != ldiskfs ] &&
@@ -8887,7 +8671,19 @@ test_120() { # LU-11130
 }
 run_test 120 "cross-target rename should not create bad symlinks"
 
-test_122() {
+test_121(){
+	stopall
+	start_mgsmds || error "MGS MDS Start failed"
+	fail mgs
+	stop_mds || error "Stopping MDSes failed"
+	#failback
+	start_mds
+	fail mgs
+	stop_mds || error "Stopping MDSes failed"
+}
+run_test 121 "failover MGS"
+
+test_122a() {
 	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
 	[[ "$OST1_VERSION" -ge $(version_code 2.11.53) ]] ||
 		skip "Need OST version at least 2.11.53"
@@ -8906,7 +8702,70 @@ test_122() {
 
 	cleanup
 }
-run_test 122 "Check OST sequence update"
+run_test 122a "Check OST sequence update"
+
+test_122b() {
+	[[ "$OST1_VERSION" -ge $(version_code 2.11.53) ]] ||
+		skip "Need OST version at least 2.11.53"
+	local err
+
+	reformat
+	LOAD_MODULES_REMOTE=true load_modules
+#define OBD_FAIL_OFD_SET_OID 0x1e0
+	do_facet ost1 $LCTL set_param fail_loc=0x00001e0
+
+	stack_trap cleanup EXIT
+	setup_noconfig
+	do_facet ost1 $LCTL set_param obdfilter.*.precreate_batch=256
+	$LFS mkdir -i0 -c1 $DIR/$tdir || error "failed to create directory"
+	$LFS setstripe -i0 -c1 $DIR/$tdir || error "failed to setstripe"
+	do_facet ost1 $LCTL set_param fail_loc=0
+	# overflow IDIF 32bit and create > OST_MAX_PRECREATE*5
+	# so a new wrong sequence would differ from an original with error
+	#define OST_MAX_PRECREATE 20000
+	local ost_max_precreate=20100
+	local num_create=$(( ost_max_precreate * 5 ))
+
+	# Check the number of inodes available on OST0
+	local files=0
+	local ifree=$($LFS df -i $MOUNT |
+		awk '/OST0000/ { print $4 }'; exit ${PIPESTATUS[0]})
+
+	log "On OST0, $ifree inodes available. Want $num_create. rc=$?"
+
+	if [ $ifree -lt 10000 ]; then
+		files=$(( ifree - 50 ))
+	else
+		files=10000
+	fi
+
+	local j=$((num_create / files + 1))
+
+	for i in $(seq 1 $j); do
+		createmany -o $DIR/$tdir/$tfile-$i- $files ||
+			error "createmany fail create $files files: $?"
+		unlinkmany $DIR/$tdir/$tfile-$i- $files ||
+			error "unlinkmany failed unlink $files files"
+	done
+	sync
+	touch $DIR/$tdir/$tfile
+	do_facet ost1 sync
+	#we need a write req during recovery for ofd_seq_load
+	replay_barrier ost1
+	dd if=/dev/urandom of=$DIR/$tdir/$tfile bs=1024k count=1 oflag=sync ||
+		 error "failed to write file"
+
+	# OBD_FAIL_OST_CREATE_NET 0x204
+	do_facet ost1 $LCTL set_param fail_loc=0x80000204
+	fail ost1
+	createmany -o $DIR/$tdir/file_ 100
+	sync
+
+	err=$(do_facet ost1 dmesg | tac | sed "/Recovery over/,$ d" |
+	      grep "OST replaced or reformatted")
+	[ -z "$err" ] || error $err
+}
+run_test 122b "Check OST sequence wouldn't change when IDIF 32bit overflows"
 
 test_123aa() {
 	remote_mgs_nodsh && skip "remote MGS with nodsh"
@@ -8937,11 +8796,15 @@ test_123ab() {
 
 	local yaml
 	local orig_val
+	local mgs_arg=""
+
+	[[ $MGS_VERSION -gt $(version_code 2.13.54) ]] ||
+		mgs_arg="--device MGS"
 
 	orig_val=$(do_facet mgs $LCTL get_param jobid_name)
 	do_facet mgs $LCTL set_param -P jobid_name="TESTNAME"
 
-	yaml=$(do_facet mgs $LCTL --device MGS llog_print params |
+	yaml=$(do_facet mgs $LCTL $mgs_arg llog_print params |
 	       grep jobid_name | tail -n 1)
 
 	local param=$(awk '{ print $10 }' <<< "$yaml")
@@ -8960,12 +8823,16 @@ test_123ac() { # LU-11566
 
 	local start=10
 	local end=50
+	local mgs_arg=""
+
+	[[ $MGS_VERSION -gt $(version_code 2.13.54) ]] ||
+		mgs_arg="--device MGS"
 
 	[ -d $MOUNT/.lustre ] || setup
 
 	# - { index: 10, event: add_uuid, nid: 192.168.20.1@tcp(0x20000c0a81401,
 	#     node: 192.168.20.1@tcp }
-	do_facet mgs $LCTL --device MGS \
+	do_facet mgs $LCTL $mgs_arg \
 		llog_print --start $start --end $end $FSNAME-client | tr -d , |
 		while read DASH BRACE INDEX idx EVENT BLAH BLAH BLAH; do
 		(( idx >= start )) || error "llog_print index $idx < $start"
@@ -9001,30 +8868,43 @@ test_123ad() { # LU-11566
 run_test 123ad "llog_print shows all records"
 
 test_123ae() { # LU-11566
+	local max
+	local mgs_arg=""
+	local log
+	local id
+	local orig
+	local new
+	local rpcs
+
 	remote_mgs_nodsh && skip "remote MGS with nodsh"
 	[ -d $MOUNT/.lustre ] || setupall
 
-	local max=$($LCTL get_param -n osc.*-OST0000-*.max_dirty_mb | head -1)
+	max=$($LCTL get_param -n osc.*-OST0000-*.max_dirty_mb | head -1)
+	pgs=$($LCTL get_param -n osc.*-OST0000-*.max_pages_per_rpc | head -1)
+	[[ $MGS_VERSION -gt $(version_code 2.13.54) ]] ||
+		mgs_arg="--device MGS"
 
 	if do_facet mgs "$LCTL help llog_cancel" 2>&1| grep -q -- --log_id; then
 		# save one set_param -P record in case none exist
-		do_facet mgs $LCTL set_param -P osc.*.max_dirty_mb=$max
 
-		local log=params
-		local orig=$(do_facet mgs $LCTL --device MGS llog_print $log |
-			     tail -1 | awk '{ print $4 }' | tr -d , )
+		do_facet mgs $LCTL set_param -P osc.*.max_pages_per_rpc=$pgs
+		stack_trap "do_facet mgs $LCTL set_param -P -d \
+				osc.*.max_pages_per_rpc"
+
+		log=params
+		orig=$(do_facet mgs $LCTL $mgs_arg llog_print $log |
+			tail -1 | awk '{ print $4 }' | tr -d , )
 		do_facet mgs $LCTL set_param -P osc.*.max_dirty_mb=$max
-		do_facet mgs $LCTL --device MGS llog_print $log | tail -1 |
+		do_facet mgs $LCTL $mgs_arg llog_print $log | tail -1 |
 			grep "parameter: osc.*.max_dirty_mb" ||
 			error "new set_param -P wasn't stored in params log"
 
 		# - { index: 71, event: set_param, device: general,
 		#     param: osc.*.max_dirty_mb, value: 256 }
-		local id=$(do_facet mgs $LCTL --device MGS llog_print $log |
-			   tail -1 | awk '{ print $4 }' | tr -d , )
-
-		do_facet mgs $LCTL --device MGS llog_cancel $log --log_idx=$id
-		local new=$(do_facet mgs $LCTL --device MGS llog_print $log |
+		id=$(do_facet mgs $LCTL $mgs_arg llog_print $log |
+		     tail -1 | awk '{ print $4 }' | tr -d , )
+		do_facet mgs $LCTL $mgs_arg llog_cancel $log --log_idx=$id
+		local new=$(do_facet mgs $LCTL $mgs_arg llog_print $log |
 			    tail -1 | awk '{ print $4 }' | tr -d , )
 		(( new == orig )) ||
 			error "new llog_cancel now $new, not at $orig records"
@@ -9033,18 +8913,25 @@ test_123ae() { # LU-11566
 	# test old positional parameters for a while still
 	if [ "$MGS_VERSION" -le $(version_code 3.1.53) ]; then
 		log=$FSNAME-client
+
+		do_facet mgs $LCTL conf_param \
+			$FSNAME-OST0000.osc.max_pages_per_rpc=$pgs
+		stack_trap "do_facet mgs $LCTL conf_param -d \
+				$FSNAME-OST0000.osc.max_pages_per_rpc"
+
 		orig=$(do_facet mgs $LCTL --device MGS llog_print $log |
 		       tail -1 | awk '{ print $4 }' | tr -d , )
 		do_facet mgs $LCTL conf_param $FSNAME-OST0000.osc.max_dirty_mb=$max
 		do_facet mgs $LCTL --device MGS llog_print $log |
 			tail -1 | grep "parameter: osc.max_dirty_mb" ||
 			error "old conf_param wasn't stored in params log"
-
+		do_facet mgs $LCTL --device MGS llog_print $log
 		# - { index: 71, event: conf_param, device: testfs-OST0000-osc,
 		#     param: osc.max_dirty_mb=256 }
 		id=$(do_facet mgs $LCTL --device MGS llog_print $log |
 		     tail -1 | awk '{ print $4 }' | tr -d , )
 		do_facet mgs $LCTL --device MGS llog_cancel $log $id
+		do_facet mgs $LCTL --device MGS llog_print $log
 		new=$(do_facet mgs $LCTL --device MGS llog_print $log |
 		      tail -1 | awk '{ print $4 }' | tr -d , )
 		(( new == orig )) ||
@@ -9053,6 +8940,85 @@ test_123ae() { # LU-11566
 }
 run_test 123ae "llog_cancel can cancel requested record"
 
+test_123af() { #LU-13609
+	[ "$MGS_VERSION" -ge $(version_code 2.13.54) -a \
+	   "$MDS1_VERSION" -ge $(version_code 2.13.54) ] ||
+		skip "Need both MGS and MDS version at least 2.13.54"
+
+	[ -d $MOUNT/.lustre ] || setupall
+	stack_trap "do_facet mds1 $LCTL set_param fail_loc=0" EXIT
+
+	local device
+	local facet
+	local cmd
+	local orig_clist
+	local orig_count
+	local new_clist
+	local new_count
+
+	for device in "MGS" "$FSNAME-MDT0000"; do
+		cmd="--device $device llog_catlist"
+		echo "lctl $cmd ..."
+		if [ "$device" = "MGS" ]; then
+			facet="mgs"
+		else
+			facet="mds1"
+		fi
+		orig_clist=($(do_facet $facet $LCTL $cmd | awk '{ print $2 }'))
+		orig_count=${#orig_clist[@]}
+		echo "orig_clist: ${orig_clist[@]}"
+
+		#define OBD_FAIL_CATLIST 0x131b
+		#fetch to llog records from the second one
+		do_facet $facet $LCTL set_param fail_loc=0x131b fail_val=2
+
+		new_clist=($(do_facet $facet $LCTL $cmd | awk '{ print $2 }'))
+		new_count=${#new_clist[@]}
+		echo "new_clist: ${new_clist[@]}"
+
+		[ $new_count -eq $((orig_count - 1)) ] ||
+			error "$new_count != $orig_count - 1"
+		for i in $(seq 0 $new_count); do
+			j=$((i + 1))
+			[ "${orig_clist[$j]}" = "${new_clist[$i]}" ] ||
+				error "${orig_clist[$j]} != ${new_clist[$i]}"
+		done
+		do_facet mds1 $LCTL set_param fail_loc=0
+		echo "done"
+	done
+}
+run_test 123af "llog_catlist can show all config files correctly"
+
+test_123ag() { # LU-15142
+	local rec
+	local orig_val
+
+	remote_mgs_nodsh && skip "remote MGS with nodsh"
+	(( $MGS_VERSION >= $(version_code 2.14.55) )) ||
+		skip "Need server version least 2.14.55"
+
+	[ -d $MOUNT/.lustre ] || setup
+
+	orig_val=$(do_facet mgs $LCTL get_param jobid_name)
+	stack_trap "do_facet mgs $LCTL set_param -P jobid_name=$orig_val"
+
+	do_facet mgs $LCTL set_param -P jobid_name="TESTNAME1"
+	do_facet mgs $LCTL set_param -P -d jobid_name
+	rec=$(do_facet mgs $LCTL --device MGS llog_print params |
+		grep -c jobid_name)
+	(( rec == 0 )) || error "parameter was not deleted, check #1"
+	do_facet mgs $LCTL set_param -P jobid_name="TESTNAME1"
+	rec=$(do_facet mgs $LCTL --device MGS llog_print params |
+		grep -c jobid_name)
+	(( rec == 1)) || error "parameter is not set"
+	# usage with ordinary set_param format works too
+	do_facet mgs $LCTL set_param -P -d jobid_name="ANY"
+	rec=$(do_facet mgs $LCTL --device MGS llog_print params |
+		grep -c jobid_name)
+	(( rec == 0 )) || error "parameter was not deleted, check #2"
+}
+run_test 123ag "llog_print skips values deleted by set_param -P -d"
+
 test_123F() {
 	remote_mgs_nodsh && skip "remote MGS with nodsh"
 
@@ -9075,10 +9041,7 @@ test_123F() {
 	echo "Writeconf"
 	writeconf_all
 	echo "Remounting"
-	mountmgs
-	mountmds
-	mountoss
-	mountcli
+	setup_noconfig
 
 	# Reapply the config from before
 	echo "Setting configuration parameters"
@@ -9101,6 +9064,12 @@ test_124()
 	[ -z $mds2failover_HOST ] && skip "needs MDT failover setup"
 
 	setup
+	do_facet mgs $LCTL --device MGS llog_print $FSNAME-client |
+		grep 1.2.3.4@tcp && error "Should not be fake nid"
+	do_facet mgs $LCTL conf_param $FSNAME-MDT0001.failover.node=1.2.3.4@tcp\
+		|| error "Set params error"
+	do_facet mgs $LCTL --device MGS llog_print $FSNAME-client |
+		grep 1.2.3.4@tcp || error "Fake nid should be added"
 	cleanup
 
 	load_modules
@@ -9239,7 +9208,7 @@ test_125()
 
 		echo "Before: ${facet} ${device} ${pre_max_sectors} ${max_hw_sectors}"
 
-		do_facet ${facet} "libtool execute l_tunedisk ${device}"
+		do_facet ${facet} "l_tunedisk ${device}"
 
 		# Value after l_tunedisk
 		local post_max_sectors=$(get_max_sectors_kb ${facet} ${device})
@@ -9286,6 +9255,170 @@ test_126() {
 }
 run_test 126 "mount in parallel shouldn't cause a crash"
 
+test_127() {
+	[[ "$ost1_FSTYPE" == ldiskfs ]] || skip "ldiskfs only test"
+
+	cleanup
+	setup
+	zconf_umount_clients $RCLIENTS $MOUNT
+
+	wait_osp_active ost ${FSNAME}-OST0000 0 1
+	local osc_tgt="$FSNAME-OST0000-osc-$($LFS getname -i $DIR)"
+	local avail1=($($LCTL get_param -n osc.${osc_tgt}.kbytesavail))
+
+	$LFS setstripe -i 0 $DIR/$tfile || error "failed creating $DIR/$tfile"
+	dd if=/dev/zero of=$DIR/$tfile bs=1M oflag=direct || true
+
+	local avail2=($($LCTL get_param -n osc.${osc_tgt}.kbytesavail))
+
+	if ((avail2 * 100 / avail1 > 1)); then
+		lfs df $DIR
+		ls -l $DIR/$tfile
+		error "more than 1% space left: before=$avail1 after=$avail2"
+	fi
+
+	local mbs=$(($(stat -c %s $DIR/$tfile) / (1024 * 1024)))
+
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=$mbs conv=notrunc \
+		oflag=direct || error "overwrite failed"
+}
+run_test 127 "direct io overwrite on full ost"
+
+test_128()
+{
+	combined_mgs_mds && skip "need separate mgs device"
+	[ "$ost2_FSTYPE" == zfs ] && import_zpool ost2
+
+	format_ost 2
+	# Try to apply nolocallogs to the virgin OST. Should fail.
+	do_facet ost2 "$TUNEFS --nolocallogs $(ostdevname 2)" &&
+		error "nolocallogs should not be allowed on the virgin target"
+
+	setupall
+	stopall
+
+	[ "$ost1_FSTYPE" == zfs ] && import_zpool ost1
+	# Start OST without MGS (local configs)
+	do_facet ost1 "$TUNEFS --dryrun $(ostdevname 1)"
+	start_ost || error "unable to start OST1"
+	stop_ost || error "Unable to stop OST1"
+
+	[ "$ost1_FSTYPE" == zfs ] && import_zpool ost1
+	# Do not allow reading local configs, should fail
+	do_facet ost1 "$TUNEFS --nolocallogs $(ostdevname 1)" ||
+		error "Can not set nolocallogs"
+	start_ost && error "OST1 started, but should fail"
+
+	# Connect to MGS successfully, reset nolocallogs flag
+	[ "$ost1_FSTYPE" == zfs ] && import_zpool ost1
+	start_mgs || error "unable to start MGS"
+	start_ost || error "unable to start OST1"
+
+	do_facet ost1 "$TUNEFS --dryrun $(ostdevname 1)" | grep "nolocallogs" &&
+		error "nolocallogs expected to be reset"
+
+	stop_ost || error "Unable to stop OST1"
+}
+run_test 128 "Force using remote logs with --nolocallogs"
+
+test_129()
+{
+	stopall
+	start_mds || error "MDS start failed"
+	format_ost 1
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS &&
+		error "start ost1 should fail" || true
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS &&
+		error "second start ost1 should fail" || true
+	do_facet ost1 "$TUNEFS --writeconf $(ostdevname 1)"
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
+		error "start ost1 failed"
+	stop ost1
+	stop_mds
+}
+run_test 129 "attempt to connect an OST with the same index should fail"
+
+test_130()
+{
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	setupall
+	stop_mdt 2 || error "mdt2 stop failed"
+	do_facet mds2 "$TUNEFS --writeconf $(mdsdevname 2)"
+	start_mdt 2 || error "mdt2 start failed"
+	do_facet mds2 "$LCTL dl" | grep MDT0001-osp-MDT0001 &&
+		error "Illegal OSP device created" || true
+}
+run_test 130 "re-register an MDT after writeconf"
+
+test_131() {
+	[ "$mds1_FSTYPE" == "ldiskfs" ] || skip "ldiskfs only test"
+	do_facet mds1 $DEBUGFS -R features $(mdsdevname 1) |
+		grep -q project || skip "skip project quota not supported"
+
+	local projid
+
+	setupall
+	test_mkdir -c $MDSCOUNT -p $DIR/$tdir
+	$LFS project -p 1000 $DIR/$tdir || error "set dir project id failed"
+	createmany -o $DIR/$tdir/f 512
+	for ((i = 0; i < 512; ++i)); do
+		$LFS project -p $i $DIR/$tdir/f${i} ||
+			error "set f${i} project id failed"
+	done
+
+	test_mkdir -c $MDSCOUNT -p $DIR/$tdir.inherit
+	$LFS project -p 1001 -s $DIR/$tdir.inherit
+	createmany -o $DIR/$tdir.inherit/f 128
+	(( $($LFS project $DIR/$tdir.inherit/f* |
+		awk '$1 == 1001 { print }' | wc -l) == 128 )) ||
+			error "files did not inherit projid 1001"
+
+	stopall
+
+	for i in $(seq $MDSCOUNT); do
+		mds_backup_restore mds$i ||
+			error "Backup/restore on mds$i failed"
+	done
+
+	setupall
+
+	projid=($($LFS project -d $DIR/$tdir))
+	[ ${projid[0]} == "1000" ] ||
+		error "projid expected 1000 not ${projid[0]}"
+	for ((i = 0; i < 512; ++i)); do
+		projid=($($LFS project $DIR/$tdir/f${i}))
+		[ ${projid[0]} == "$i" ] ||
+			error "projid expected $i not ${projid[0]}"
+	done
+
+	(( $($LFS project $DIR/$tdir.inherit/f* |
+		awk '$1 == 1001 { print }' | wc -l) == 128 )) ||
+			error "restore did not copy projid 1001"
+}
+run_test 131 "MDT backup restore with project ID"
+
+test_132() {
+	local err_cnt
+	local err_cnt2
+
+	reformat
+	combined_mgs_mds || start_mgs || error "unable to start MGS"
+	start_mdt 1 || error "unable to start mdt1"
+
+	err_cnt=$(do_facet mds1 dmesg | grep -c "cannot take the layout locks")
+	stop_mdt 1 || error "stop mdt1 failed"
+
+	[ "$mds1_FSTYPE" == zfs ] && import_zpool mds1
+	do_facet mds1 $TUNEFS --param mdt.hsm_control=enabled $(mdsdevname 1) ||
+		error "tunefs failed"
+	start_mdt 1 || error "cannot start mdt1"
+
+	err_cnt2=$(do_facet mds1 dmesg | grep -c "cannot take the layout locks")
+	[ $err_cnt -eq $err_cnt2 ] || error "Can not take the layout lock"
+	stop_mdt 1 || error "stop mdt1 failed"
+}
+run_test 132 "hsm_actions processed after failover"
+
 if ! combined_mgs_mds ; then
 	stop mgs
 fi