X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity.sh;h=fb4539de903f22579a741729e6a2ad291842aec1;hp=4a6ad1227347ff7c83efc490bebf713da2c3f424;hb=cfed34264ac3ab96507a55973e6ffe1d648ca527;hpb=a0a812d2b019b97356b0d6a1a8debd7d46fed00b

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 4a6ad12..fb4539d 100755
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -12,9 +12,8 @@ ONLY=${ONLY:-"$*"}
 ALWAYS_EXCEPT="                42a  42b  42c  42d  45   68b   $SANITY_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
-# with LOD/OSP landing
-# bug number for skipped tests: LU-2036 LU-8411
-ALWAYS_EXCEPT="                 76	407 $ALWAYS_EXCEPT"
+# bug number for skipped tests: LU-2036 LU-8411 LU-9096
+ALWAYS_EXCEPT="                 76	407     253 $ALWAYS_EXCEPT"
 
 is_sles11()						# LU-4341
 {
@@ -75,12 +74,12 @@ init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}
 init_logging
 
-#                                  5              12          (min)"
-[ "$SLOW" = "no" ] && EXCEPT_SLOW="24D 27m 64b 68 71 115 300o"
+#                                  5          12          (min)"
+[ "$SLOW" = "no" ] && EXCEPT_SLOW="27m 64b 68 71 115 300o"
 
 if [ $(facet_fstype $SINGLEMDS) = "zfs" ]; then
-	# bug number for skipped test: LU-4536 LU-1957
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT  65ic    180"
+	# bug number for skipped test: LU-9514 LU-4536 LU-1957
+	ALWAYS_EXCEPT="$ALWAYS_EXCEPT  51f     65ic    180"
 	#                                               13    (min)"
 	[ "$SLOW" = "no" ] && EXCEPT_SLOW="$EXCEPT_SLOW 51b"
 fi
@@ -1038,53 +1037,73 @@ page_size() {
 }
 
 simple_cleanup_common() {
+	local rc=0
 	trap 0
+	[ -z "$DIR" -o -z "$tdir" ] && return 0
+
+	local start=$SECONDS
 	rm -rf $DIR/$tdir
+	rc=$?
 	wait_delete_completed
+	echo "cleanup time $((SECONDS - start))"
+	return $rc
 }
 
 max_pages_per_rpc() {
-	$LCTL get_param -n mdc.*.max_pages_per_rpc | head -n1
+	local mdtname="$(printf "MDT%04x" ${1:-0})"
+	$LCTL get_param -n mdc.*$mdtname*.max_pages_per_rpc
 }
 
 test_24v() {
-	local NRFILES=100000
-	local FREE_INODES=$(mdt_free_inodes 0)
-	[[ $FREE_INODES -lt $NRFILES ]] &&
-		skip "not enough free inodes $FREE_INODES required $NRFILES" &&
-		return
-
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-	trap simple_cleanup_common EXIT
-
+	local nrfiles=${COUNT:-100000}
 	# Performance issue on ZFS see LU-4072 (c.f. LU-2887)
-	[ $(facet_fstype $SINGLEMDS) = "zfs" ] && NRFILES=10000
+	[ $(facet_fstype $SINGLEMDS) = "zfs" ] && nrfiles=${COUNT:-10000}
 
-	test_mkdir -p $DIR/$tdir
-	createmany -m $DIR/$tdir/$tfile $NRFILES
+	local fname="$DIR/$tdir/$tfile"
+	test_mkdir "$(dirname $fname)"
+	# assume MDT0000 has the fewest inodes
+	local stripes=$($LFS getdirstripe -c $(dirname $fname))
+	local free_inodes=$(($(mdt_free_inodes 0) * stripes))
+	[[ $free_inodes -lt $nrfiles ]] && nrfiles=$free_inodes
+
+	trap simple_cleanup_common EXIT
+
+	createmany -m "$fname" $nrfiles
 
 	cancel_lru_locks mdc
 	lctl set_param mdc.*.stats clear
 
-	ls $DIR/$tdir >/dev/null || error "error in listing large dir"
-
+	# was previously test_24D: LU-6101
+	# readdir() returns correct number of entries after cursor reload
+	local num_ls=$(ls $DIR/$tdir | wc -l)
+	local num_uniq=$(ls $DIR/$tdir | sort -u | wc -l)
+	local num_all=$(ls -a $DIR/$tdir | wc -l)
+	if [ $num_ls -ne $nrfiles -o $num_uniq -ne $nrfiles -o \
+	     $num_all -ne $((nrfiles + 2)) ]; then
+		error "Expected $nrfiles files, got $num_ls " \
+			"($num_uniq unique $num_all .&..)"
+	fi
 	# LU-5 large readdir
-	# DIRENT_SIZE = 32 bytes for sizeof(struct lu_dirent) +
-	#               8 bytes for name(filename is mostly 5 in this test) +
-	#               8 bytes for luda_type
+	# dirent_size = 32 bytes for sizeof(struct lu_dirent) +
+	#               N bytes for name (len($nrfiles) rounded to 8 bytes) +
+	#               8 bytes for luda_type (4 bytes rounded to 8 bytes)
 	# take into account of overhead in lu_dirpage header and end mark in
-	# each page, plus one in RPC_NUM calculation.
-	DIRENT_SIZE=48
-	RPC_SIZE=$(($(max_pages_per_rpc) * $(page_size)))
-	RPC_NUM=$(((NRFILES * DIRENT_SIZE + RPC_SIZE - 1) / RPC_SIZE + 1))
-	mds_readpage=$(lctl get_param mdc.*MDT0000*.stats |
-				awk '/^mds_readpage/ {print $2}')
-	[[ $mds_readpage -gt $RPC_NUM ]] &&
-		error "large readdir doesn't take effect"
+	# each page, plus one in rpc_num calculation.
+	local dirent_size=$((32 + (${#tfile} | 7) + 1 + 8))
+	local page_entries=$((($(page_size) - 24) / dirent_size))
+	local mdt_idx=$($LFS getdirstripe -i $(dirname $fname))
+	local rpc_pages=$(max_pages_per_rpc $mdt_idx)
+	local rpc_max=$((nrfiles / (page_entries * rpc_pages) + stripes))
+	local mds_readpage=$(calc_stats mdc.*.stats mds_readpage)
+	echo "readpages: $mds_readpage rpc_max: $rpc_max"
+	(( $mds_readpage < $rpc_max - 2 || $mds_readpage > $rpc_max + 1)) &&
+		error "large readdir doesn't take effect: " \
+		      "$mds_readpage should be about $rpc_max"
 
 	simple_cleanup_common
 }
-run_test 24v "list directory with large files (handle hash collision, bug: 17560)"
+run_test 24v "list large directory (test hash collision, b=17560)"
 
 test_24w() { # bug21506
         SZ1=234852
@@ -1163,6 +1182,7 @@ test_24A() { # LU-3182
 
 	rm -rf $DIR/$tdir
 	test_mkdir -p $DIR/$tdir
+	trap simple_cleanup_common EXIT
 	createmany -m $DIR/$tdir/$tfile $NFILES
 	local t=$(ls $DIR/$tdir | wc -l)
 	local u=$(ls $DIR/$tdir | sort -u | wc -l)
@@ -1171,7 +1191,7 @@ test_24A() { # LU-3182
 		error "Expected $NFILES files, got $t ($u unique $v .&..)"
 	fi
 
-	rm -rf $DIR/$tdir || error "Can not delete directories"
+	simple_cleanup_common || error "Can not delete directories"
 }
 run_test 24A "readdir() returns correct number of entries."
 
@@ -1229,23 +1249,6 @@ test_24C() {
 }
 run_test 24C "check .. in striped dir"
 
-test_24D() { # LU-6101
-	local NFILES=50000
-
-	rm -rf $DIR/$tdir
-	mkdir -p $DIR/$tdir
-	createmany -m $DIR/$tdir/$tfile $NFILES
-	local t=$(ls $DIR/$tdir | wc -l)
-	local u=$(ls $DIR/$tdir | sort -u | wc -l)
-	local v=$(ls -ai $DIR/$tdir | sort -u | wc -l)
-	if [ $t -ne $NFILES -o $u -ne $NFILES -o $v -ne $((NFILES + 2)) ] ; then
-		error "Expected $NFILES files, got $t ($u unique $v .&..)"
-	fi
-
-	rm -rf $DIR/$tdir || error "Can not delete directories"
-}
-run_test 24D "readdir() returns correct number of entries after cursor reload"
-
 test_24E() {
 	[[ $MDSCOUNT -lt 4 ]] && skip "needs >= 4 MDTs" && return
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
@@ -1661,7 +1664,7 @@ test_27u() { # bug 4900
 #define OBD_FAIL_MDS_OSC_PRECREATE      0x139
 	do_nodes $list $LCTL set_param fail_loc=0x139
 	test_mkdir -p $DIR/$tdir
-	rm -rf $DIR/$tdir/*
+	trap simple_cleanup_common EXIT
 	createmany -o $DIR/$tdir/t- 1000
 	do_nodes $list $LCTL set_param fail_loc=0
 
@@ -1669,6 +1672,7 @@ test_27u() { # bug 4900
 	$GETSTRIPE $DIR/$tdir > $TLOG
 	OBJS=$(awk -vobj=0 '($1 == 0) { obj += 1 } END { print obj; }' $TLOG)
 	unlinkmany $DIR/$tdir/t- 1000
+	trap 0
 	[[ $OBJS -gt 0 ]] &&
 		error "$OBJS objects created on OST-0. See $TLOG" || pass
 }
@@ -1884,10 +1888,11 @@ check_seq_oid()
 		# update too, until that use mount/ll_decode_filter_fid/mount.
 		# Re-enable when debugfs will understand new filter_fid.
 		#
-		if false && [ $(facet_fstype ost$ost) == ldiskfs ]; then
+		if [ $(facet_fstype ost$ost) == ldiskfs ]; then
 			ff=$(do_facet ost$ost "$DEBUGFS -c -R 'stat $obj_file' \
 				$dev 2>/dev/null" | grep "parent=")
-		else
+		fi
+		if [ -z "$ff" ]; then
 			stop ost$ost
 			mount_fstype ost$ost
 			ff=$(do_facet ost$ost $LL_DECODE_FILTER_FID \
@@ -1903,45 +1908,39 @@ check_seq_oid()
 
 		# /mnt/O/0/d23/23: objid=23 seq=0 parent=[0x200000400:0x1e:0x1]
 		# fid: objid=23 seq=0 parent=[0x200000400:0x1e:0x0] stripe=1
-		local ff_parent=$(echo $ff|sed -e 's/.*parent=.//')
-		local ff_pseq=$(echo $ff_parent | cut -d: -f1)
-		local ff_poid=$(echo $ff_parent | cut -d: -f2)
+		#
+		# fid: parent=[0x200000400:0x1e:0x0] stripe=1 stripe_count=2 \
+		#	stripe_size=1048576 component_id=1 component_start=0 \
+		#	component_end=33554432
+		local ff_parent=$(sed -e 's/.*parent=.//' <<<$ff)
+		local ff_pseq=$(cut -d: -f1 <<<$ff_parent)
+		local ff_poid=$(cut -d: -f2 <<<$ff_parent)
 		local ff_pstripe
-		if echo $ff_parent | grep -q 'stripe='; then
-			ff_pstripe=$(echo $ff_parent | sed -e 's/.*stripe=//')
-			if echo $ff_pstripe | grep -q 'stripe_size='; then
-				ff_pstripe=$(echo $ff_pstripe | cut -d' ' -f1)
-			fi
+		if grep -q 'stripe=' <<<$ff; then
+			ff_pstripe=$(sed -e 's/.*stripe=//' -e 's/ .*//' <<<$ff)
 		else
-			#
 			# $LL_DECODE_FILTER_FID does not print "stripe="; look
-			# into f_ver in this case.  See the comment on
-			# ff_parent.
-			#
-			ff_pstripe=$(echo $ff_parent | cut -d: -f3 |
-				sed -e 's/\]//')
+			# into f_ver in this case.  See comment on ff_parent.
+			ff_pstripe=$(cut -d: -f3 <<<$ff_parent | sed -e 's/]//')
 		fi
 
-		if echo $ff_parent | grep -q 'stripe_count='; then
-			local ff_scnt=$(echo $ff_parent |
-					sed -e 's/.*stripe_count=//' |
-					cut -d' ' -f1)
-
-			[ $lmm_count -eq $ff_scnt ] ||
+		if grep -q 'stripe_count=' <<<$ff; then
+			local ff_scnt=$(sed -e 's/.*stripe_count=//' \
+					    -e 's/ .*//' <<<$ff)
+			[ $lmm_count = $ff_scnt ] ||
 				error "FF stripe count $lmm_count != $ff_scnt"
 		fi
-
-                # compare lmm_seq and filter_fid->ff_parent.f_seq
-                [ $ff_pseq = $lmm_seq ] ||
-                        error "FF parent SEQ $ff_pseq != $lmm_seq"
-                # compare lmm_object_id and filter_fid->ff_parent.f_oid
-                [ $ff_poid = $lmm_oid ] ||
-                        error "FF parent OID $ff_poid != $lmm_oid"
+		# compare lmm_seq and filter_fid->ff_parent.f_seq
+		[ $ff_pseq = $lmm_seq ] ||
+			error "FF parent SEQ $ff_pseq != $lmm_seq"
+		# compare lmm_object_id and filter_fid->ff_parent.f_oid
+		[ $ff_poid = $lmm_oid ] ||
+			error "FF parent OID $ff_poid != $lmm_oid"
 		(($ff_pstripe == $stripe_nr)) ||
-                        error "FF stripe $ff_pstripe != $stripe_nr"
+			error "FF stripe $ff_pstripe != $stripe_nr"
 
-                stripe_nr=$((stripe_nr + 1))
-        done
+		stripe_nr=$((stripe_nr + 1))
+	done
 }
 
 test_27z() {
@@ -2064,8 +2063,10 @@ test_27D() {
 	pool_add_targets $POOL $ost_range || error "pool_add_targets failed"
 
 	local skip27D
-	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
-		skip27D = "-s 29,30,31"
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.8.55) ] &&
+		skip27D += "-s 29"
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.9.55) ] &&
+		skip27D += "-s 30,31"
 	llapi_layout_test -d$DIR/$tdir -p$POOL -o$OSTCOUNT $skip27D ||
 		error "llapi_layout_test failed"
 
@@ -2105,11 +2106,10 @@ test_27E() {
 run_test 27E "check that default extended attribute size properly increases"
 
 test_27F() { # LU-5346/LU-7975
-
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-
 	[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.8.51) ]] &&
 		skip "Need MDS version at least 2.8.51" && return
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
 
 	test_mkdir -p $DIR/$tdir
 	rm -f $DIR/$tdir/f0
@@ -4383,6 +4383,8 @@ test_53() {
 run_test 53 "verify that MDS and OSTs agree on pre-creation ===="
 
 test_54a() {
+	perl -MSocket -e ';' || { skip "no Socket perl module installed" && return; }
+
 	$SOCKETSERVER $DIR/socket ||
 		error "$SOCKETSERVER $DIR/socket failed: $?"
 	$SOCKETCLIENT $DIR/socket ||
@@ -4844,8 +4846,22 @@ run_test 56r "check lfs find -size works =========================="
 
 test_56s() { # LU-611
 	TDIR=$DIR/${tdir}s
-	setup_56 $NUMFILES $NUMDIRS "-c $OSTCOUNT"
 
+	#LU-9369
+	setup_56 0 $NUMDIRS
+	for i in $(seq 1 $NUMDIRS); do
+		$SETSTRIPE -c $((OSTCOUNT + 1)) $TDIR/dir$i/$tfile
+	done
+	EXPECTED=$NUMDIRS
+	CMD="$LFIND -c $OSTCOUNT $TDIR"
+	NUMS=$($CMD | wc -l)
+	[ $NUMS -eq $EXPECTED ] || {
+		$GETSTRIPE -R $TDIR
+		error "\"$CMD\" wrong: found $NUMS, expected $EXPECTED"
+	}
+	rm -rf $TDIR
+
+	setup_56 $NUMFILES $NUMDIRS "-c $OSTCOUNT"
 	if [[ $OSTCOUNT -gt 1 ]]; then
 		$SETSTRIPE -c 1 $TDIR/$tfile.{0,1,2,3}
 		ONESTRIPE=4
@@ -4898,6 +4914,21 @@ run_test 56s "check lfs find -stripe-count works"
 
 test_56t() { # LU-611
 	TDIR=$DIR/${tdir}t
+
+	#LU-9369
+	setup_56 0 $NUMDIRS
+	for i in $(seq 1 $NUMDIRS); do
+		$SETSTRIPE -S 4M $TDIR/dir$i/$tfile
+	done
+	EXPECTED=$NUMDIRS
+	CMD="$LFIND -S 4M $TDIR"
+	NUMS=$($CMD | wc -l)
+	[ $NUMS -eq $EXPECTED ] || {
+		$GETSTRIPE -R $TDIR
+		error "\"$CMD\" wrong: found $NUMS, expected $EXPECTED"
+	}
+	rm -rf $TDIR
+
 	setup_56 $NUMFILES $NUMDIRS "--stripe-size 512k"
 
 	$SETSTRIPE -S 256k $TDIR/$tfile.{0,1,2,3}
@@ -6102,6 +6133,96 @@ test_77b() { # bug 10889
 }
 run_test 77b "checksum error on client write, read"
 
+cleanup_77c() {
+	trap 0
+	set_checksums 0
+	$LCTL set_param osc.*osc-[^mM]*.checksum_dump=0
+	$check_ost &&
+		do_facet ost1 $LCTL set_param obdfilter.*-OST*.checksum_dump=0
+	[ -n $osc_file_prefix ] && rm -f ${osc_file_prefix}*
+	$check_ost && [ -n $ost_file_prefix ] &&
+		do_facet ost1 rm -f ${ost_file_prefix}\*
+}
+
+test_77c() {
+	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
+	$GSS && skip "could not run with gss" && return
+
+	local bad1
+	local osc_file_prefix
+	local osc_file
+	local check_ost=false
+	local ost_file_prefix
+	local ost_file
+	local orig_cksum
+	local dump_cksum
+	local fid
+
+	# ensure corruption will occur on first OSS/OST
+	$LFS setstripe -i 0 $DIR/$tfile
+
+	[ ! -f $F77_TMP ] && setup_f77
+	dd if=$F77_TMP of=$DIR/$tfile bs=1M count=$F77SZ conv=sync ||
+		error "dd write error: $?"
+	fid=$($LFS path2fid $DIR/$tfile)
+
+	if [ $(lustre_version_code ost1) -ge $(version_code 2.5.42.6) ]
+	then
+		check_ost=true
+		ost_file_prefix=$(do_facet ost1 $LCTL get_param -n debug_path)
+		ost_file_prefix=${ost_file_prefix}-checksum_dump-ost-\\${fid}
+	else
+		echo "OSS do not support bulk pages dump upon error"
+	fi
+
+	osc_file_prefix=$($LCTL get_param -n debug_path)
+	osc_file_prefix=${osc_file_prefix}-checksum_dump-osc-\\${fid}
+
+	trap cleanup_77c EXIT
+
+	set_checksums 1
+	# enable bulk pages dump upon error on Client
+	$LCTL set_param osc.*osc-[^mM]*.checksum_dump=1
+	# enable bulk pages dump upon error on OSS
+	$check_ost &&
+		do_facet ost1 $LCTL set_param obdfilter.*-OST*.checksum_dump=1
+
+	# flush Client cache to allow next read to reach OSS
+	cancel_lru_locks osc
+
+	#define OBD_FAIL_OSC_CHECKSUM_RECEIVE       0x408
+	$LCTL set_param fail_loc=0x80000408
+	dd if=$DIR/$tfile of=/dev/null bs=1M || error "dd read error: $?"
+	$LCTL set_param fail_loc=0
+
+	rm -f $DIR/$tfile
+
+	# check cksum dump on Client
+	osc_file=$(ls ${osc_file_prefix}*)
+	[ -n "$osc_file" ] || error "no checksum dump file on Client"
+	# OBD_FAIL_OSC_CHECKSUM_RECEIVE corrupts with "bad1" at start of file
+	bad1=$(dd if=$osc_file bs=1 count=4 2>/dev/null) || error "dd error: $?"
+	[ $bad1 == "bad1" ] || error "unexpected corrupt pattern"
+	orig_cksum=$(dd if=$F77_TMP bs=1 skip=4 count=1048572 2>/dev/null |
+		     cksum)
+	dump_cksum=$(dd if=$osc_file bs=1 skip=4 2>/dev/null | cksum)
+	[[ "$orig_cksum" == "$dump_cksum" ]] ||
+		error "dump content does not match on Client"
+
+	$check_ost || skip "No need to check cksum dump on OSS"
+
+	# check cksum dump on OSS
+	ost_file=$(do_facet ost1 ls ${ost_file_prefix}\*)
+	[ -n "$ost_file" ] || error "no checksum dump file on OSS"
+	orig_cksum=$(dd if=$F77_TMP bs=1048576 count=1 2>/dev/null | cksum)
+	dump_cksum=$(do_facet ost1 dd if=$ost_file 2>/dev/null \| cksum)
+	[[ "$orig_cksum" == "$dump_cksum" ]] ||
+		error "dump content does not match on OSS"
+
+	cleanup_77c
+}
+run_test 77c "checksum error on client read with debug"
+
 test_77d() { # bug 10889
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
 	$GSS && skip "could not run with gss" && return
@@ -7603,31 +7724,113 @@ test_110() {
 }
 run_test 110 "filename length checking"
 
+#
+# Purpose: To verify dynamic thread (OSS) creation.
+#
 test_115() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
-	OSTIO_pre=$(ps -e | grep ll_ost_io | awk '{ print $4 }'| sort -n |
-		tail -1 | cut -c11-20)
-	[ -z "$OSTIO_pre" ] && skip "no OSS threads" && return
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
+	# Lustre does not stop service threads once they are started.
+	# Reset number of running threads to default.
+	stopall
+	setupall
+
+	local OSTIO_pre
+	local save_params="$TMP/sanity-$TESTNAME.parameters"
+
+	# Get ll_ost_io count before I/O
+	OSTIO_pre=$(do_facet ost1 \
+		"$LCTL get_param ost.OSS.ost_io.threads_started | cut -d= -f2")
+	# Exit if lustre is not running (ll_ost_io not running).
+	[ -z "$OSTIO_pre" ] && error "no OSS threads"
+
 	echo "Starting with $OSTIO_pre threads"
+	local thread_max=$((OSTIO_pre * 2))
+	local rpc_in_flight=$((thread_max * 2))
+	# Number of I/O Process proposed to be started.
+	local nfiles
+	local facets=$(get_facets OST)
 
-	NUMTEST=20000
-	NUMFREE=$(df -i -P $DIR | tail -n 1 | awk '{ print $4 }')
-	[[ $NUMFREE -lt $NUMTEST ]] && NUMTEST=$(($NUMFREE - 1000))
-	echo "$NUMTEST creates/unlinks"
-	test_mkdir -p $DIR/$tdir
-	createmany -o $DIR/$tdir/$tfile $NUMTEST
-	unlinkmany $DIR/$tdir/$tfile $NUMTEST
+	save_lustre_params client \
+		"osc.*OST*.max_rpcs_in_flight" > $save_params
+	save_lustre_params $facets \
+		"ost.OSS.ost_io.threads_max" >> $save_params
+
+	# Set in_flight to $rpc_in_flight
+	$LCTL set_param osc.*OST*.max_rpcs_in_flight=$rpc_in_flight ||
+		error "Failed to set max_rpcs_in_flight to $rpc_in_flight"
+	nfiles=${rpc_in_flight}
+	# Set ost thread_max to $thread_max
+	do_facet ost1 \
+		"$LCTL set_param ost.OSS.ost_io.threads_max=$thread_max"
+
+	# 5 Minutes should be sufficient for max number of OSS
+	# threads(thread_max) to be created.
+	local timeout=300
+
+	# Start I/O.
+	local WTL=${WTL:-"$LUSTRE/tests/write_time_limit"}
+	mkdir -p $DIR/$tdir
+	for i in $(seq $nfiles); do
+		local file=$DIR/$tdir/${tfile}-$i
+		$LFS setstripe -c -1 -i 0 $file
+		($WTL $file $timeout)&
+	done
 
-	OSTIO_post=$(ps -e | grep ll_ost_io | awk '{ print $4 }' | sort -n |
-		tail -1 | cut -c11-20)
+	# I/O Started - Wait for thread_started to reach thread_max or report
+	# error if thread_started is more than thread_max.
+	echo "Waiting for thread_started to reach thread_max"
+	local thread_started=0
+	local end_time=$((SECONDS + timeout))
+
+	while [ $SECONDS -le $end_time ] ; do
+		echo -n "."
+		# Get ost i/o thread_started count.
+		thread_started=$(do_facet ost1 \
+			"$LCTL get_param \
+			ost.OSS.ost_io.threads_started | cut -d= -f2")
+		# Break out if thread_started is equal/greater than thread_max
+		if [[ $thread_started -ge $thread_max ]]; then
+			echo ll_ost_io thread_started $thread_started, \
+				equal/greater than thread_max $thread_max
+			break
+		fi
+		sleep 1
+	done
+
+	# Cleanup - We have the numbers, Kill i/o jobs if running.
+	jobcount=($(jobs -p))
+	for i in $(seq 0 $((${#jobcount[@]}-1)))
+	do
+		kill -9 ${jobcount[$i]}
+		if [ $? -ne 0 ] ; then
+			echo Warning: \
+			Failed to Kill \'WTL\(I/O\)\' with pid ${jobcount[$i]}
+		fi
+	done
 
-	# don't return an error
-	[ $OSTIO_post == $OSTIO_pre ] && echo \
-	    "WARNING: No new ll_ost_io threads were created ($OSTIO_pre)" &&
-	    echo "This may be fine, depending on what ran before this test" &&
-	    echo "and how fast this system is." && return
+	# Cleanup files left by WTL binary.
+	for i in $(seq $nfiles); do
+		local file=$DIR/$tdir/${tfile}-$i
+		rm -rf $file
+		if [ $? -ne 0 ] ; then
+			echo "Warning: Failed to delete file $file"
+		fi
+	done
 
-	echo "Started with $OSTIO_pre threads, ended with $OSTIO_post"
+	restore_lustre_params <$save_params
+	rm -f $save_params || echo "Warning: delete file '$save_params' failed"
+
+	# Error out if no new thread has started or Thread started is greater
+	# than thread max.
+	if [[ $thread_started -le $OSTIO_pre ||
+			$thread_started -gt $thread_max ]]; then
+		error "ll_ost_io: thread_started $thread_started" \
+		      "OSTIO_pre $OSTIO_pre, thread_max $thread_max." \
+		      "No new thread started or thread started greater " \
+		      "than thread_max."
+	fi
 }
 run_test 115 "verify dynamic thread creation===================="
 
@@ -8977,18 +9180,16 @@ set_dir_limits () {
 	local canondev
 	local node
 
-	local LDPROC=/proc/fs/ldiskfs
+	local ldproc=/proc/fs/ldiskfs
 	local facets=$(get_facets MDS)
 
 	for facet in ${facets//,/ }; do
 		canondev=$(ldiskfs_canon \
 			   *.$(convert_facet2label $facet).mntdev $facet)
-		do_facet $facet "test -e $LDPROC/$canondev/max_dir_size" ||
-						LDPROC=/sys/fs/ldiskfs
-		do_facet $facet "echo $1 >$LDPROC/$canondev/max_dir_size"
-		do_facet $facet "test -e $LDPROC/$canondev/warning_dir_size" ||
-						LDPROC=/sys/fs/ldiskfs
-		do_facet $facet "echo $2 >$LDPROC/$canondev/warning_dir_size"
+		do_facet $facet "test -e $ldproc/$canondev/max_dir_size" ||
+			ldproc=/sys/fs/ldiskfs
+		do_facet $facet "echo $1 >$ldproc/$canondev/max_dir_size"
+		do_facet $facet "echo $2 >$ldproc/$canondev/warning_dir_size"
 	done
 }
 
@@ -9012,62 +9213,54 @@ test_129() {
 	remote_mds_nodsh && skip "remote MDS with nodsh" && return
 	local ENOSPC=28
 	local EFBIG=27
-	local has_warning=0
+	local has_warning=false
 
 	rm -rf $DIR/$tdir
 	mkdir -p $DIR/$tdir
 
 	# block size of mds1
-	local MDT_DEV=$(mdsdevname ${SINGLEMDS//mds/})
-	local MDSBLOCKSIZE=$($LCTL get_param -n mdc.*MDT0000*.blocksize)
-	local MAX=$((MDSBLOCKSIZE * 5))
-	set_dir_limits $MAX $MAX
-	local I=$(stat -c%s "$DIR/$tdir")
-	local J=0
-	while [[ $I -le $MAX ]]; do
-		$MULTIOP $DIR/$tdir/$J Oc
+	local maxsize=$(($($LCTL get_param -n mdc.*MDT0000*.blocksize) * 5))
+	set_dir_limits $maxsize $maxsize
+	local dirsize=$(stat -c%s "$DIR/$tdir")
+	local nfiles=0
+	while [[ $dirsize -le $maxsize ]]; do
+		$MULTIOP $DIR/$tdir/file_base_$nfiles Oc
 		rc=$?
-		if [ $has_warning -eq 0 ]; then
-			check_mds_dmesg '"is approaching"' &&
-				has_warning=1
+		if ! $has_warning; then
+			check_mds_dmesg '"is approaching"' && has_warning=true
 		fi
-		#check two errors ENOSPC for new version of ext4 max_dir_size patch
-		#mainline kernel commit df981d03eeff7971ac7e6ff37000bfa702327ef1
-		#and EFBIG for previous versions
+		# check two errors:
+		# ENOSPC for new ext4 max_dir_size (kernel commit df981d03ee)
+		# EFBIG for previous versions included in ldiskfs series
 		if [ $rc -eq $EFBIG -o $rc -eq $ENOSPC ]; then
 			set_dir_limits 0 0
 			echo "return code $rc received as expected"
 
-			createmany -o $DIR/$tdir/$J_file_ 1000 ||
+			createmany -o $DIR/$tdir/file_extra_$nfiles. 5 ||
 				error_exit "create failed w/o dir size limit"
 
 			check_mds_dmesg '"has reached"' ||
-				error_exit "has reached message should be output"
+				error_exit "reached message should be output"
 
 			[ $has_warning -eq 0 ] &&
 				error_exit "warning message should be output"
 
-			I=$(stat -c%s "$DIR/$tdir")
+			dirsize=$(stat -c%s "$DIR/$tdir")
 
-			if [ $(lustre_version_code $SINGLEMDS) -lt \
-					$(version_code 2.4.51) ]
-			then
-				[[ $I -eq $MAX ]] && return 0
-			else
-				[[ $I -gt $MAX ]] && return 0
-			fi
-			error_exit "current dir size $I, previous limit $MAX"
+			[[ $dirsize -ge $maxsize ]] && return 0
+			error_exit "current dir size $dirsize, " \
+				   "previous limit $maxsize"
 		elif [ $rc -ne 0 ]; then
 			set_dir_limits 0 0
-			error_exit "return code $rc received instead of expected " \
-				   "$EFBIG or $ENOSPC, files in dir $I"
+			error_exit "return $rc received instead of expected " \
+				   "$EFBIG or $ENOSPC, files in dir $dirsize"
 		fi
-		J=$((J+1))
-		I=$(stat -c%s "$DIR/$tdir")
+		nfiles=$((nfiles + 1))
+		dirsize=$(stat -c%s "$DIR/$tdir")
 	done
 
 	set_dir_limits 0 0
-	error "exceeded dir size limit $MAX($MDSCOUNT) : $I bytes"
+	error "exceeded dir size limit $maxsize($MDSCOUNT) : $dirsize bytes"
 }
 run_test 129 "test directory size limit ========================"
 
@@ -11066,6 +11259,8 @@ test_160d() {
 run_test 160d "verify that changelog log catch the migrate event"
 
 test_160e() {
+	remote_mds_nodsh && skip "remote MDS with nodsh" && return
+
 	# Create a user
 	CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0 \
 		changelog_register -n)
@@ -11632,12 +11827,11 @@ test_180b() {
 	local rc=0
 	local rmmod_remote=0
 
-	do_facet ost1 "lsmod | grep -q obdecho || "                      \
-		      "{ insmod ${LUSTRE}/obdecho/obdecho.ko || "        \
-		      "modprobe obdecho; }" && rmmod_remote=1
+	do_rpc_nodes $(facet_active_host ost1) load_module obdecho/obdecho &&
+		rmmod_remote=true || error "failed to load module obdecho"
 	target=$(do_facet ost1 $LCTL dl | awk '/obdfilter/ {print $4;exit}')
 	[[ -n $target ]] && { obdecho_test $target ost1 || rc=1; }
-	[ $rmmod_remote -eq 1 ] && do_facet ost1 "rmmod obdecho"
+	$rmmod_remote && do_facet ost1 "rmmod obdecho"
 	return $rc
 }
 run_test 180b "test obdecho directly on obdfilter"
@@ -11868,14 +12062,16 @@ test_184d() {
 		error "create $file2 failed"
 	$OPENFILE -f O_CREAT:O_LOV_DELAY_CREATE $file3 ||
 		error "create $file3 failed"
-	lovea1=$($LFS getstripe $file1 | sed 1d)
+	lovea1=$(get_layout_param $file1)
 
 	$LFS swap_layouts $file2 $file3 ||
 		error "swap $file2 $file3 layouts failed"
 	$LFS swap_layouts $file1 $file2 ||
 		error "swap $file1 $file2 layouts failed"
 
-	lovea2=$($LFS getstripe $file2 | sed 1d)
+	lovea2=$(get_layout_param $file2)
+	echo "$lovea1"
+	echo "$lovea2"
 	[ "$lovea1" == "$lovea2" ] || error "lovea $lovea1 != $lovea2"
 
 	lovea1=$(getfattr -n trusted.lov $file1 | grep ^trusted)
@@ -12059,11 +12255,11 @@ default_attr() {
 check_default_stripe_attr() {
 	ACTUAL=$($GETSTRIPE $* $DIR/$tdir)
 	case $1 in
-	--stripe-count|--count)
+	--stripe-count|-c)
 		[ -n "$2" ] && EXPECTED=0 || EXPECTED=$(default_attr count);;
-	--stripe-size|--size)
+	--stripe-size|-S)
 		[ -n "$2" ] && EXPECTED=0 || EXPECTED=$(default_attr size);;
-	--stripe-index|--index)
+	--stripe-index|-i)
 		EXPECTED=-1;;
 	*)
 		error "unknown getstripe attr '$1'"
@@ -12640,32 +12836,34 @@ test_216() { # bug 20317
 		"ldlm.namespaces.filter-*.contended_locks" >> $p
 	save_lustre_params $facets \
 		"ldlm.namespaces.filter-*.contention_seconds" >> $p
-	clear_osc_stats
+	clear_stats osc.*.osc_stats
 
-        # agressive lockless i/o settings
-        for node in $(osts_nodes); do
-                do_node $node 'lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes 2000000; lctl set_param -n ldlm.namespaces.filter-*.contended_locks 0; lctl set_param -n ldlm.namespaces.filter-*.contention_seconds 60'
-        done
-        lctl set_param -n osc.*.contention_seconds 60
+	# agressive lockless i/o settings
+	do_nodes $(comma_list $(osts_nodes)) \
+		"lctl set_param -n ldlm.namespaces.*.max_nolock_bytes=2000000 \
+			ldlm.namespaces.filter-*.contended_locks=0 \
+			ldlm.namespaces.filter-*.contention_seconds=60"
+	lctl set_param -n osc.*.contention_seconds=60
 
-        $DIRECTIO write $DIR/$tfile 0 10 4096
-        $CHECKSTAT -s 40960 $DIR/$tfile
+	$DIRECTIO write $DIR/$tfile 0 10 4096
+	$CHECKSTAT -s 40960 $DIR/$tfile
 
-        # disable lockless i/o
-        for node in $(osts_nodes); do
-                do_node $node 'lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes 0; lctl set_param -n ldlm.namespaces.filter-*.contended_locks 32; lctl set_param -n ldlm.namespaces.filter-*.contention_seconds 0'
-        done
-        lctl set_param -n osc.*.contention_seconds 0
-        clear_osc_stats
+	# disable lockless i/o
+	do_nodes $(comma_list $(osts_nodes)) \
+		"lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes=0 \
+			ldlm.namespaces.filter-*.contended_locks=32 \
+			ldlm.namespaces.filter-*.contention_seconds=0"
+	lctl set_param -n osc.*.contention_seconds=0
+	clear_stats osc.*.osc_stats
 
-        dd if=/dev/zero of=$DIR/$tfile count=0
-        $CHECKSTAT -s 0 $DIR/$tfile
+	dd if=/dev/zero of=$DIR/$tfile count=0
+	$CHECKSTAT -s 0 $DIR/$tfile
 
-        restore_lustre_params <$p
-        rm -f $p
-        rm $DIR/$tfile
+	restore_lustre_params <$p
+	rm -f $p
+	rm $DIR/$tfile
 }
-run_test 216 "check lockless direct write works and updates file size and kms correctly"
+run_test 216 "check lockless direct write updates file size and kms correctly"
 
 test_217() { # bug 22430
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
@@ -14432,70 +14630,93 @@ ladvise_no_ioctl()
 	return 1
 }
 
+percent() {
+	bc <<<"scale=2; ($1 - $2) * 100 / $2"
+}
+
+# run a random read IO workload
+# usage: random_read_iops <filename> <filesize> <iosize>
+random_read_iops() {
+	local file=$1
+	local fsize=$2
+	local iosize=${3:-4096}
+
+	$READS -f $file -s $fsize -b $iosize -n $((fsize / iosize)) -t 60 |
+		sed -e '/^$/d' -e 's#.*s, ##' -e 's#MB/s##'
+}
+
+drop_file_oss_cache() {
+	local file="$1"
+	local nodes="$2"
+
+	$LFS ladvise -a dontneed $file 2>/dev/null ||
+		do_nodes $nodes "echo 3 > /proc/sys/vm/drop_caches"
+}
+
 ladvise_willread_performance()
 {
 	local repeat=10
+	local average_origin=0
 	local average_cache=0
 	local average_ladvise=0
 
 	for ((i = 1; i <= $repeat; i++)); do
 		echo "Iter $i/$repeat: reading without willread hint"
 		cancel_lru_locks osc
-		do_nodes $(comma_list $(osts_nodes)) \
-			"echo 3 > /proc/sys/vm/drop_caches"
-		local speed_origin=$($READS -f $DIR/$tfile -s $size \
-			-b 4096 -n $((size / 4096)) -t 60 |
-			sed -e '/^$/d' -e 's#.*s, ##' -e 's#MB/s##')
+		drop_file_oss_cache $DIR/$tfile $(comma_list $(osts_nodes))
+		local speed_origin=$(random_read_iops $DIR/$tfile $size)
+		echo "Iter $i/$repeat: uncached speed: $speed_origin"
+		average_origin=$(bc <<<"$average_origin + $speed_origin")
 
-		echo "Iter $i/$repeat: Reading again without willread hint"
 		cancel_lru_locks osc
-		local speed_cache=$($READS -f $DIR/$tfile -s $size \
-			-b 4096 -n $((size / 4096)) -t 60 |
-			sed -e '/^$/d' -e 's#.*s, ##' -e 's#MB/s##')
+		local speed_cache=$(random_read_iops $DIR/$tfile $size)
+		echo "Iter $i/$repeat: OSS cache speed: $speed_cache"
+		average_cache=$(bc <<<"$average_cache + $speed_cache")
 
-		echo "Iter $i/$repeat: reading with willread hint"
 		cancel_lru_locks osc
-		do_nodes $(comma_list $(osts_nodes)) \
-			"echo 3 > /proc/sys/vm/drop_caches"
-		lfs ladvise -a willread $DIR/$tfile ||
-			error "Ladvise failed"
-		local speed_ladvise=$($READS -f $DIR/$tfile -s $size \
-			-b 4096 -n $((size / 4096)) -t 60 |
-			sed -e '/^$/d' -e 's#.*s, ##' -e 's#MB/s##')
-
-		local cache_speedup=$(echo "scale=2; \
-			($speed_cache-$speed_origin)/$speed_origin*100" | bc)
-		cache_speedup=$(echo ${cache_speedup%.*})
-		echo "Iter $i/$repeat: cache speedup: $cache_speedup%"
-		average_cache=$((average_cache + cache_speedup))
-
-		local ladvise_speedup=$(echo "scale=2; \
-			($speed_ladvise-$speed_origin)/$speed_origin*100" | bc)
-		ladvise_speedup=$(echo ${ladvise_speedup%.*})
-		echo "Iter $i/$repeat: ladvise speedup: $ladvise_speedup%"
-		average_ladvise=$((average_ladvise + ladvise_speedup))
+		drop_file_oss_cache $DIR/$tfile $(comma_list $(osts_nodes))
+		$LFS ladvise -a willread $DIR/$tfile || error "ladvise failed"
+		local speed_ladvise=$(random_read_iops $DIR/$tfile $size)
+		echo "Iter $i/$repeat: ladvise speed: $speed_ladvise"
+		average_ladvise=$(bc <<<"$average_ladvise + $speed_ladvise")
 	done
-	average_cache=$((average_cache / repeat))
-	average_ladvise=$((average_ladvise / repeat))
-
-	if [ $average_cache -lt 20 ]; then
-		echo "Speedup with cache is less than 20% ($average_cache%),"\
-			"skipping check of speedup with willread:"\
-			"$average_ladvise%"
+	average_origin=$(bc <<<"scale=2; $average_origin / $repeat")
+	average_cache=$(bc <<<"scale=2; $average_cache / $repeat")
+	average_ladvise=$(bc <<<"scale=2; $average_ladvise / $repeat")
+
+	speedup_cache=$(percent $average_cache $average_origin)
+	speedup_ladvise=$(percent $average_ladvise $average_origin)
+
+	echo "Average uncached read: $average_origin"
+	echo "Average speedup with OSS cached read: " \
+		"$average_cache = +$speedup_cache%"
+	echo "Average speedup with ladvise willread: " \
+		"$average_ladvise = +$speedup_ladvise%"
+
+	local lowest_speedup=20
+	if [ ${average_cache%.*} -lt $lowest_speedup ]; then
+		echo "Speedup with OSS cached read less than $lowest_speedup%, "
+			"got $average_cache%. Skipping ladvise willread check."
 		return 0
 	fi
 
-	local lowest_speedup=$((average_cache / 2))
-	[ $average_ladvise -gt $lowest_speedup ] ||
+	# the test won't work on ZFS until it supports 'ladvise dontneed', but
+	# it is still good to run until then to exercise 'ladvise willread'
+	! $LFS ladvise -a dontneed $DIR/$tfile &&
+		[ "$(facet_fstype ost1)" = "zfs" ] &&
+		echo "osd-zfs does not support dontneed or drop_caches" &&
+		return 0
+
+	lowest_speedup=$(bc <<<"scale=2; $average_cache / 2")
+	[ ${average_ladvise%.*} -gt $lowest_speedup ] ||
 		error_not_in_vm "Speedup with willread is less than " \
-			   "$lowest_speedup%, got $average_ladvise%"
-	echo "Speedup with willread ladvise: $average_ladvise%"
-	echo "Speedup with cache: $average_cache%"
+			"$lowest_speedup%, got $average_ladvise%"
 }
 
 test_255a() {
 	[ $(lustre_version_code ost1) -lt $(version_code 2.8.54) ] &&
 		skip "lustre < 2.8.54 does not support ladvise " && return
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
 
 	lfs setstripe -c -1 -i 0 $DIR/$tfile || error "$tfile failed"
 
@@ -14580,6 +14801,10 @@ facet_meminfo() {
 }
 
 test_255b() {
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
+	lfs setstripe -c 1 -i 0 $DIR/$tfile
+
 	ladvise_no_type dontneed $DIR/$tfile &&
 		skip "dontneed ladvise is not supported" && return
 
@@ -14589,10 +14814,9 @@ test_255b() {
 	[ $(lustre_version_code ost1) -lt $(version_code 2.8.54) ] &&
 		skip "lustre < 2.8.54 does not support ladvise" && return
 
-	[ "$(facet_fstype ost1)" = "zfs" ] &&
-		skip "zfs-osd does not support dontneed advice" && return
-
-	lfs setstripe -c 1 -i 0 $DIR/$tfile
+	! $LFS ladvise -a dontneed $DIR/$tfile &&
+		[ "$(facet_fstype ost1)" = "zfs" ] &&
+		skip "zfs-osd does not support 'ladvise dontneed'" && return
 
 	local size_mb=100
 	local size=$((size_mb * 1048576))
@@ -15296,6 +15520,8 @@ test_300n() {
 		skip "Need MDS version at least 2.7.55" && return
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
 	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+	remote_mds_nodsh && skip "remote MDS with nodsh" && return
+
 	local stripe_index
 	local list=$(comma_list $(mdts_nodes))
 
@@ -15488,6 +15714,7 @@ test_311() {
 	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.8.54) ] &&
 		skip "lustre < 2.8.54 does not contain LU-4825 fix" && return
 	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
+	remote_mds_nodsh && skip "remote MDS with nodsh" && return
 
 	local old_iused=$($LFS df -i | grep OST0000 | awk '{ print $3 }')
 
@@ -15541,7 +15768,7 @@ zfs_oid_to_objid()
 	local objid=$2
 
 	local vdevdir=$(dirname $(facet_vdevice $ost))
-	local cmd="$ZDB -e -p $vdevdir -dddd $(facet_device $ost)"
+	local cmd="$ZDB -e -p $vdevdir -ddddd $(facet_device $ost)"
 	local zfs_zapid=$(do_facet $ost $cmd |
 			  grep -w "/O/0/d$((objid%32))" -C 5 |
 			  awk '/Object/{getline; print $1}')
@@ -15570,12 +15797,15 @@ zfs_object_blksz() {
 }
 
 test_312() { # LU-4856
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
 	[ $(facet_fstype ost1) = "zfs" ] ||
 		{ skip "the test only applies to zfs" && return; }
 
 	local max_blksz=$(do_facet ost1 \
 			  $ZFS get -p recordsize $(facet_device ost1) |
 			  awk '!/VALUE/{print $3}')
+	local min_blksz=$(getconf PAGE_SIZE)
 
 	# to make life a little bit easier
 	$LFS mkdir -c 1 -i 0 $DIR/$tdir
@@ -15590,7 +15820,7 @@ test_312() { # LU-4856
 
 	# block size change by sequential over write
 	local blksz
-	for ((bs=4096; bs <= max_blksz; bs <<= 2)); do
+	for ((bs=$min_blksz; bs <= max_blksz; bs <<= 2)); do
 		dd if=/dev/zero of=$tf bs=$bs count=1 oflag=sync conv=notrunc
 
 		blksz=$(zfs_object_blksz ost1 $zfs_objid)
@@ -15599,18 +15829,18 @@ test_312() { # LU-4856
 	rm -f $tf
 
 	# block size change by sequential append write
-	dd if=/dev/zero of=$tf bs=4K count=1 oflag=sync conv=notrunc
+	dd if=/dev/zero of=$tf bs=$min_blksz count=1 oflag=sync conv=notrunc
 	oid=$($LFS getstripe $tf | awk '/obdidx/{getline; print $2}')
 	zfs_objid=$(zfs_oid_to_objid ost1 $oid)
 
-	for ((count = 1; count < $((max_blksz / 4096)); count *= 2)); do
-		dd if=/dev/zero of=$tf bs=4K count=$count seek=$count \
+	for ((count = 1; count < $((max_blksz / min_blksz)); count *= 2)); do
+		dd if=/dev/zero of=$tf bs=$min_blksz count=$count seek=$count \
 			oflag=sync conv=notrunc
 
 		blksz=$(zfs_object_blksz ost1 $zfs_objid)
-		blksz=$((blksz / 8192)) # in 2*4K unit
-		[ $blksz -eq $count ] ||
-			error "blksz error(in 8k): $blksz, expected: $count"
+		[ $blksz -eq $((2 * count * min_blksz)) ] ||
+			error "blksz error, actual $blksz, "	\
+				"expected: 2 * $count * $min_blksz"
 	done
 	rm -f $tf
 
@@ -15619,9 +15849,10 @@ test_312() { # LU-4856
 	oid=$($LFS getstripe $tf | awk '/obdidx/{getline; print $2}')
 	zfs_objid=$(zfs_oid_to_objid ost1 $oid)
 
-	dd if=/dev/zero of=$tf bs=8K count=1 oflag=sync conv=notrunc
+	dd if=/dev/zero of=$tf bs=1K count=1 oflag=sync conv=notrunc
 	blksz=$(zfs_object_blksz ost1 $zfs_objid)
-	[ $blksz -eq 8192 ] || error "blksz error: $blksz, expected: 8k"
+	[ $blksz -eq $min_blksz ] ||
+		error "blksz error: $blksz, expected: $min_blksz"
 
 	dd if=/dev/zero of=$tf bs=64K count=1 oflag=sync conv=notrunc seek=128
 	blksz=$(zfs_object_blksz ost1 $zfs_objid)
@@ -15634,6 +15865,8 @@ test_312() { # LU-4856
 run_test 312 "make sure ZFS adjusts its block size by write pattern"
 
 test_313() {
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
 	local file=$DIR/$tfile
 	rm -f $file
 	$SETSTRIPE -c 1 -i 0 $file || error "setstripe failed"
@@ -15706,12 +15939,15 @@ test_fake_rw() {
 	rm -f $DIR/$tfile
 }
 test_399a() { # LU-7655 for OST fake write
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
 	test_fake_rw write
 }
 run_test 399a "fake write should not be slower than normal write"
 
-
 test_399b() { # LU-8726 for OST fake read
+	remote_ost_nodsh && skip "remote OST with nodsh" && return
+
 	if [ "$(facet_fstype ost1)" != "ldiskfs" ]; then
 		skip "only for ldiskfs" && return 0
 	fi
@@ -15921,6 +16157,10 @@ test_403() {
 run_test 403 "i_nlink should not drop to zero due to aliasing"
 
 test_404() { # LU-6601
+	local server_version=$(lustre_version_code $SINGLEMDS)
+	[[ $server_version -ge $(version_code 2.8.53) ]] ||
+		{ skip "Need server version newer than 2.8.52"; return 0; }
+
 	remote_mds_nodsh && skip "remote MDS with nodsh" && return
 	local mosps=$(do_facet $SINGLEMDS $LCTL dl |
 		awk '/osp .*-osc-MDT/ { print $4}')
@@ -16039,9 +16279,9 @@ run_test 406 "DNE support fs default striping"
 
 test_407() {
 	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
-
 	[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.8.55) ]] &&
 		skip "Need MDS version at least 2.8.55" && return
+	remote_mds_nodsh && skip "remote MDS with nodsh" && return
 
 	$LFS mkdir -i 0 -c 1 $DIR/$tdir.0 ||
 		error "$LFS mkdir -i 0 -c 1 $tdir.0 failed"
@@ -16113,10 +16353,10 @@ test_409()
 run_test 409 "Large amount of cross-MDTs hard links on the same file"
 
 prep_801() {
+	[[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] ||
+	[[ $(lustre_version_code ost1) -lt $(version_code 2.9.55) ]] &&
+		skip "Need server version at least 2.9.55" & exit 0
 	start_full_debug_logging
-	# cleanup unused barrier locks before test
-	do_facet mgs $LCTL barrier_rescan $FSNAME ||
-		error "Fail to prep barrier test env"
 }
 
 post_801() {
@@ -16324,6 +16564,11 @@ cleanup_802() {
 }
 
 test_802() {
+
+	[[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] ||
+	[[ $(lustre_version_code ost1) -lt $(version_code 2.9.55) ]] &&
+		skip "Need server version at least 2.9.55" & exit 0
+
 	mkdir $DIR/$tdir || error "(1) fail to mkdir"
 
 	cp $LUSTRE/tests/test-framework.sh $DIR/$tdir/ ||