X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=b3238821df225fcf8f74d1953dbbe40615eec1c7;hp=baaaae907eeafc4fe3c3484139c0ff925dbfea60;hb=9c4156e6fc146a198bb342e28eb246f1076889bd;hpb=5a3dfc2b5d9027ec04774cda69ef01d5026f1740

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index baaaae9..b323882 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -54,16 +54,25 @@ fi
 [ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] &&
     MODPROBECONF=/etc/modprobe.conf
 
+sanitize_parameters() {
+	for i in DIR DIR1 DIR2 MOUNT MOUNT1 MOUNT2
+	do
+		local path=${!i}
+		if [ -d "$path" ]; then
+			eval export $i=$(echo $path | sed -r 's/\/+$//g')
+		fi
+	done
+}
 assert_DIR () {
-    local failed=""
-    [[ $DIR/ = $MOUNT/* ]] || \
-        { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; }
-    [[ $DIR1/ = $MOUNT1/* ]] || \
-        { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; }
-    [[ $DIR2/ = $MOUNT2/* ]] || \
-        { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; }
+	local failed=""
+	[[ $DIR/ = $MOUNT/* ]] ||
+		{ failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; }
+	[[ $DIR1/ = $MOUNT1/* ]] ||
+		{ failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; }
+	[[ $DIR2/ = $MOUNT2/* ]] ||
+		{ failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; }
 
-    [ -n "$failed" ] && exit 99 || true
+	[ -n "$failed" ] && exit 99 || true
 }
 
 usage() {
@@ -324,20 +333,20 @@ init_test_env() {
 	export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3}
 	export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"}
 
-    # command line
+	# command line
 
-    while getopts "rvwf:" opt $*; do
-        case $opt in
-            f) CONFIG=$OPTARG;;
-            r) REFORMAT=--reformat;;
-            v) VERBOSE=true;;
-            w) WRITECONF=writeconf;;
-            \?) usage;;
-        esac
-    done
+	while getopts "rvwf:" opt $*; do
+		case $opt in
+			f) CONFIG=$OPTARG;;
+			r) REFORMAT=yes;;
+			v) VERBOSE=true;;
+			w) WRITECONF=writeconf;;
+			\?) usage;;
+		esac
+	done
 
-    shift $((OPTIND - 1))
-    ONLY=${ONLY:-$*}
+	shift $((OPTIND - 1))
+	ONLY=${ONLY:-$*}
 
 	# print the durations of each test if "true"
 	DDETAILS=${DDETAILS:-false}
@@ -560,13 +569,15 @@ load_modules_local() {
     load_module lov/lov
     load_module mgc/mgc
     load_module obdecho/obdecho
-    if ! client_only; then
-        SYMLIST=/proc/kallsyms
-        grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
-        grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
-        grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
+	if ! client_only; then
+		SYMLIST=/proc/kallsyms
+		grep -q crc16 $SYMLIST ||
+			{ modprobe crc16 2>/dev/null || true; }
+		grep -q -w jbd2 $SYMLIST ||
+			{ modprobe jbd2 2>/dev/null || true; }
 		load_module lfsck/lfsck
-		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+		[ "$LQUOTA" != "no" ] &&
+			load_module quota/lquota $LQUOTAOPTS
 		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
 			modprobe zfs
 			load_module osd-zfs/osd_zfs
@@ -574,6 +585,8 @@ load_modules_local() {
 		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
 			grep -q exportfs_decode_fh $SYMLIST ||
 				{ modprobe exportfs 2> /dev/null || true; }
+			grep -q -w mbcache $SYMLIST ||
+				{ modprobe mbcache 2>/dev/null || true; }
 			load_module ../ldiskfs/ldiskfs
 			load_module osd-ldiskfs/osd_ldiskfs
 		fi
@@ -585,7 +598,7 @@ load_modules_local() {
 		load_module osp/osp
 		load_module ofd/ofd
 		load_module osp/osp
-    fi
+	fi
 
 	load_module llite/lustre
 	llite_lloop_enabled && load_module llite/llite_lloop
@@ -817,7 +830,7 @@ facet_type() {
 facet_number() {
 	local facet=$1
 
-	if [ $facet == mgs ]; then
+	if [ $facet == mgs ] || [ $facet == client ]; then
 		return 1
 	fi
 
@@ -1116,7 +1129,7 @@ get_osd_param() {
 	local name=$3
 
 	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
-		osd-*.$device.$name 2>&1" | grep -v 'Found no match'
+		osd-*.$device.$name 2>&1" | grep -v 'error:'
 }
 
 set_osd_param() {
@@ -1126,7 +1139,7 @@ set_osd_param() {
 	local value=$4
 
 	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
-		osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
+		osd-*.$device.$name=$value 2>&1" | grep -v 'error:'
 }
 
 set_debug_size () {
@@ -1223,6 +1236,10 @@ mount_facet() {
 	local opt=${facet}_opt
 	local mntpt=$(facet_mntpt $facet)
 	local opts="${!opt} $@"
+	local fstype=$(facet_fstype $facet)
+	local devicelabel
+
+	module_loaded lustre || load_modules
 
 	if [ $(facet_fstype $facet) == ldiskfs ] &&
 	   ! do_facet $facet test -b ${!dev}; then
@@ -1234,6 +1251,16 @@ mount_facet() {
 		import_zpool $facet || return ${PIPESTATUS[0]}
 	fi
 
+	case $fstype in
+	ldiskfs)
+		devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");;
+	zfs)
+		devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \
+						lustre:svname ${!dev}");;
+	*)
+		error "unknown fstype!";;
+	esac
+
 	echo "Starting ${facet}: $opts ${!dev} $mntpt"
 	# for testing LU-482 error handling in mount_facets() and test_0a()
 	if [ -f $TMP/test-lu482-trigger ]; then
@@ -1249,6 +1276,11 @@ mount_facet() {
 		return $RC
 	fi
 
+	health=$(do_facet ${facet} "$LCTL get_param -n health_check")
+	if [[ "$health" != "healthy" ]]; then
+		error "$facet is in a unhealthy state"
+	fi
+
 	set_default_debug_facet $facet
 
 	if [[ $facet == mds* ]]; then
@@ -1259,7 +1291,6 @@ mount_facet() {
 	if [[ $opts =~ .*nosvc.* ]]; then
 		echo "Start ${!dev} without service"
 	else
-		local fstype=$(facet_fstype $facet)
 
 		case $fstype in
 		ldiskfs)
@@ -1277,6 +1308,12 @@ mount_facet() {
 		esac
 	fi
 
+	# commit the device label change to disk
+	if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then
+		do_facet $facet "sync; sync; sync"
+	fi
+
+
 	label=$(devicelabel ${facet} ${!dev})
 	[ -z "$label" ] && echo no label for ${!dev} && exit 1
 	eval export ${facet}_svc=${label}
@@ -1287,19 +1324,19 @@ mount_facet() {
 
 # start facet device options
 start() {
-    local facet=$1
-    shift
-    local device=$1
-    shift
-    eval export ${facet}_dev=${device}
-    eval export ${facet}_opt=\"$@\"
+	local facet=$1
+	shift
+	local device=$1
+	shift
+	eval export ${facet}_dev=${device}
+	eval export ${facet}_opt=\"$@\"
 
-    local varname=${facet}failover_dev
-    if [ -n "${!varname}" ] ; then
-        eval export ${facet}failover_dev=${!varname}
-    else
-        eval export ${facet}failover_dev=$device
-    fi
+	local varname=${facet}failover_dev
+	if [ -n "${!varname}" ] ; then
+		eval export ${facet}failover_dev=${!varname}
+	else
+		eval export ${facet}failover_dev=$device
+	fi
 
 	local mntpt=$(facet_mntpt $facet)
 	do_facet ${facet} mkdir -p $mntpt
@@ -1380,70 +1417,6 @@ quota_type() {
 	return $rc
 }
 
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-#     it should be removed whenever we drop the interoperability for such
-#     server.
-restore_quota_old() {
-	local mntpt=${1:-$MOUNT}
-	local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2)
-	if [ ! "$old_QUOTA_TYPE" ] ||
-		[ "$quota_type" = "$old_QUOTA_TYPE" ]; then
-		return
-	fi
-	quota_save_version $old_QUOTA_TYPE
-}
-
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-#     it should be removed whenever we drop the interoperability for such
-#     server.
-setup_quota_old(){
-	local mntpt=$1
-
-	# no quota enforcement for now and accounting works out of the box
-	return
-
-    # We need save the original quota_type params, and restore them after testing
-
-    # Suppose that quota type the same on mds and ost
-    local quota_type=$(quota_type | grep MDT | cut -d "=" -f2)
-    [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!"
-    echo "[HOST:$HOSTNAME] [old_quota_type:$quota_type] [new_quota_type:$QUOTA_TYPE]"
-    if [ "$quota_type" != "$QUOTA_TYPE" ]; then
-        export old_QUOTA_TYPE=$quota_type
-        quota_save_version $QUOTA_TYPE
-    else
-        qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE)
-        $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type"
-    fi
-
-    local quota_usrs=$QUOTA_USERS
-
-    # get_filesystem_size
-    local disksz=$(lfs_df $mntpt | grep "summary"  | awk '{print $2}')
-    local blk_soft=$((disksz + 1024))
-    local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
-
-    local Inodes=$(lfs_df -i $mntpt | grep "summary"  | awk '{print $2}')
-    local i_soft=$Inodes
-    local i_hard=$((i_soft + i_soft / 20))
-
-    echo "Total disk size: $disksz  block-softlimit: $blk_soft block-hardlimit:
-        $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard"
-
-    local cmd
-    for usr in $quota_usrs; do
-        echo "Setting up quota on $HOSTNAME:$mntpt for $usr..."
-        for type in u g; do
-            cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt"
-            echo "+ $cmd"
-            eval $cmd || error "$cmd FAILED!"
-        done
-        # display the quota status
-        echo "Quota settings for $usr : "
-        $LFS quota -v -u $usr $mntpt || true
-    done
-}
-
 # get mdt quota type
 mdt_quota_type() {
 	local varsvc=${SINGLEMDS}_svc
@@ -1461,11 +1434,6 @@ ost_quota_type() {
 
 # restore old quota type settings
 restore_quota() {
-	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
-		restore_quota_old
-		return
-	fi
-
 	if [ "$old_MDT_QUOTA_TYPE" ]; then
 		do_facet mgs $LCTL conf_param \
 			$FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
@@ -1501,11 +1469,6 @@ mdt_free_inodes() {
 }
 
 setup_quota(){
-	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
-		setup_quota_old $1
-		return
-	fi
-
 	local mntpt=$1
 
 	# save old quota type & set new quota type
@@ -1554,25 +1517,37 @@ setup_quota(){
 }
 
 zconf_mount() {
-    local client=$1
-    local mnt=$2
-    local opts=${3:-$MOUNT_OPTS}
-    opts=${opts:+-o $opts}
-    local flags=${4:-$MOUNT_FLAGS}
-
-    local device=$MGSNID:/$FSNAME
-    if [ -z "$mnt" -o -z "$FSNAME" ]; then
-        echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt
-        exit 1
-    fi
-
-    echo "Starting client: $client: $flags $opts $device $mnt"
-    do_node $client mkdir -p $mnt
-    do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1
-
-    set_default_debug_nodes $client
+	local client=$1
+	local mnt=$2
+	local opts=${3:-$MOUNT_OPTS}
+	opts=${opts:+-o $opts}
+	local flags=${4:-$MOUNT_FLAGS}
+
+	local device=$MGSNID:/$FSNAME$FILESET
+	if [ -z "$mnt" -o -z "$FSNAME" ]; then
+		echo "Bad mount command: opt=$flags $opts dev=$device " \
+		     "mnt=$mnt"
+		exit 1
+	fi
+
+	echo "Starting client: $client: $flags $opts $device $mnt"
+	do_node $client mkdir -p $mnt
+	if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then
+		do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \
+			$mnt || return 1
+		#disable FILESET if not supported
+		do_nodes $client lctl get_param -n \
+			mdc.$FSNAME-MDT0000*.import | grep -q subtree ||
+				device=$MGSNID:/$FSNAME
+		do_node $client mkdir -p $mnt/$FILESET
+		do_node $client "! grep -q $mnt' ' /proc/mounts ||
+			umount $mnt"
+	fi
+	do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1
+
+	set_default_debug_nodes $client
 
-    return 0
+	return 0
 }
 
 zconf_umount() {
@@ -1662,21 +1637,35 @@ sanity_mount_check () {
 
 # mount clients if not mouted
 zconf_mount_clients() {
-    local clients=$1
-    local mnt=$2
-    local opts=${3:-$MOUNT_OPTS}
-    opts=${opts:+-o $opts}
-    local flags=${4:-$MOUNT_FLAGS}
-
-    local device=$MGSNID:/$FSNAME
-    if [ -z "$mnt" -o -z "$FSNAME" ]; then
-        echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt
-        exit 1
-    fi
-
-    echo "Starting client $clients: $flags $opts $device $mnt"
-
-    do_nodes $clients "
+	local clients=$1
+	local mnt=$2
+	local opts=${3:-$MOUNT_OPTS}
+	opts=${opts:+-o $opts}
+	local flags=${4:-$MOUNT_FLAGS}
+
+	local device=$MGSNID:/$FSNAME$FILESET
+	if [ -z "$mnt" -o -z "$FSNAME" ]; then
+		echo "Bad conf mount command: opt=$flags $opts dev=$device " \
+		     "mnt=$mnt"
+		exit 1
+	fi
+
+	echo "Starting client $clients: $flags $opts $device $mnt"
+	if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then
+		do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
+			umount $mnt"
+		do_nodes $clients $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \
+			$mnt || return 1
+		#disable FILESET if not supported
+		do_nodes $clients lctl get_param -n \
+			mdc.$FSNAME-MDT0000*.import | grep -q subtree ||
+				device=$MGSNID:/$FSNAME
+		do_nodes $clients mkdir -p $mnt/$FILESET
+		do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
+			umount $mnt"
+	fi
+
+	do_nodes $clients "
 running=\\\$(mount | grep -c $mnt' ');
 rc=0;
 if [ \\\$running -eq 0 ] ; then
@@ -1686,12 +1675,12 @@ if [ \\\$running -eq 0 ] ; then
 fi;
 exit \\\$rc" || return ${PIPESTATUS[0]}
 
-    echo "Started clients $clients: "
-    do_nodes $clients "mount | grep $mnt' '"
+	echo "Started clients $clients: "
+	do_nodes $clients "mount | grep $mnt' '"
 
-    set_default_debug_nodes $clients
+	set_default_debug_nodes $clients
 
-    return 0
+	return 0
 }
 
 zconf_umount_clients() {
@@ -2558,8 +2547,8 @@ replay_barrier() {
 	# inside fail() and fail_abort().
 	#
 	do_facet $facet $LCTL --device ${!svc} readonly
-	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-	$LCTL mark "local REPLAY BARRIER on ${!svc}"
+	do_facet $facet $LCTL mark "$HOSTNAME: $facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "$HOSTNAME: local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nodf() {
@@ -2569,8 +2558,8 @@ replay_barrier_nodf() {
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
 	do_facet $facet $LCTL --device ${!svc} readonly
-	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-	$LCTL mark "local REPLAY BARRIER on ${!svc}"
+	do_facet $facet $LCTL mark "$HOSTNAME: $facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "$HOSTNAME: local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nosync() {
@@ -2579,8 +2568,8 @@ replay_barrier_nosync() {
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
 	do_facet $facet $LCTL --device ${!svc} readonly
-	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-	$LCTL mark "local REPLAY BARRIER on ${!svc}"
+	do_facet $facet $LCTL mark "$HOSTNAME: $facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "$HOSTNAME: local REPLAY BARRIER on ${!svc}"
 }
 
 #
@@ -2895,26 +2884,26 @@ change_active() {
 }
 
 do_node() {
-    local verbose=false
-    # do not stripe off hostname if verbose, bug 19215
-    if [ x$1 = x--verbose ]; then
-        shift
-        verbose=true
-    fi
+	local verbose=false
+	# do not stripe off hostname if verbose, bug 19215
+	if [ x$1 = x--verbose ]; then
+		shift
+		verbose=true
+	fi
 
-    local HOST=$1
-    shift
-    local myPDSH=$PDSH
-    if [ "$HOST" = "$HOSTNAME" ]; then
-        myPDSH="no_dsh"
-    elif [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ]; then
-        echo "cannot run remote command on $HOST with $myPDSH"
-        return 128
-    fi
-    if $VERBOSE; then
-        echo "CMD: $HOST $@" >&2
-        $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
-    fi
+	local HOST=$1
+	shift
+	local myPDSH=$PDSH
+	if [ "$HOST" = "$HOSTNAME" ]; then
+		myPDSH="no_dsh"
+	elif [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ]; then
+		echo "cannot run remote command on $HOST with $myPDSH"
+		return 128
+	fi
+	if $VERBOSE; then
+		echo "CMD: $HOST $@" >&2
+		$myPDSH $HOST "$LCTL mark \"$HOSTNAME: $@\"" &>/dev/null || :
+	fi
 
     if [ "$myPDSH" = "rsh" ]; then
 # we need this because rsh does not return exit code of an executed command
@@ -3010,17 +2999,18 @@ do_nodes() {
         return $?
     fi
 
-    # This is part from do_node
-    local myPDSH=$PDSH
+	# This is part from do_node
+	local myPDSH=$PDSH
 
-    [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \
-        echo "cannot run remote command on $rnodes with $myPDSH" && return 128
+	[ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] &&
+		echo "cannot run remote command on $rnodes with $myPDSH" &&
+		return 128
 
-    export FANOUT=$(get_node_count "${rnodes//,/ }")
-    if $VERBOSE; then
-        echo "CMD: $rnodes $@" >&2
-        $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
-    fi
+	export FANOUT=$(get_node_count "${rnodes//,/ }")
+	if $VERBOSE; then
+		echo "CMD: $rnodes $@" >&2
+		$myPDSH $rnodes "$LCTL mark \"$HOSTNAME: $@\"" &>/dev/null || :
+	fi
 
     # do not replace anything from pdsh output if -N is used
     # -N     Disable hostname: prefix on lines of output.
@@ -4004,21 +3994,22 @@ is_empty_fs() {
 }
 
 check_and_setup_lustre() {
-    nfs_client_mode && return
+	sanitize_parameters
+	nfs_client_mode && return
 	cifs_client_mode && return
 
-    local MOUNTED=$(mounted_lustre_filesystems)
-
-    local do_check=true
-    # 1.
-    # both MOUNT and MOUNT2 are not mounted
-    if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
-        [ "$REFORMAT" ] && formatall
-        # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
-        setupall
-        is_mounted $MOUNT || error "NAME=$NAME not mounted"
-        export I_MOUNTED=yes
-        do_check=false
+	local MOUNTED=$(mounted_lustre_filesystems)
+
+	local do_check=true
+	# 1.
+	# both MOUNT and MOUNT2 are not mounted
+	if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
+		[ "$REFORMAT" = "yes" ] && formatall
+		# setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
+		setupall
+		is_mounted $MOUNT || error "NAME=$NAME not mounted"
+		export I_MOUNTED=yes
+		do_check=false
     # 2.
     # MOUNT2 is mounted
     elif is_mounted $MOUNT2; then
@@ -4634,33 +4625,10 @@ set_nodes_failloc () {
 }
 
 cancel_lru_locks() {
-    $LCTL mark "cancel_lru_locks $1 start"
-
-    if [ $1 != "MGC" ]; then
-	for d in $(lctl get_param -N ldlm.namespaces.*.lru_size |
-		 egrep -i $1); do
-	    $LCTL set_param -n $d=clear
-	done
-	$LCTL get_param ldlm.namespaces.*.lock_unused_count | egrep -i $1 |
-		grep -v '=0'
-    else
-	for d in $(find \
-		/{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lru_size \
-		2> /dev/null); do
-	    echo "clear" > $d
-	done
-
-	for d in $(find \
-		/{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lock_unused_count \
-		2> /dev/null); do
-	    if [ $(cat $d) != 0 ]; then
-		echo "ldlm.namespaces.$(echo "$d" |
-			cut -f 7 -d'/').lock_unused_count=$(cat $d)"
-	    fi
-	done
-    fi
-
-    $LCTL mark "cancel_lru_locks $1 stop"
+	#$LCTL mark "$HOSTNAME: cancel_lru_locks $1 start"
+	$LCTL set_param -n ldlm.namespaces.*$1*.lru_size=clear
+	$LCTL get_param ldlm.namespaces.*$1*.lock_unused_count | grep -v '=0'
+	#$LCTL mark "$HOSTNAME: cancel_lru_locks $1 stop"
 }
 
 default_lru_size()
@@ -4737,20 +4705,17 @@ stop_full_debug_logging() {
 
 # prints bash call stack
 print_stack_trace() {
+	local skip=${1:-1}
 	echo "  Trace dump:"
-	for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do
-		local s=${BASH_SOURCE[$i]}
-		local l=${BASH_LINENO[$i-1]}
-		local f=${FUNCNAME[$i]}
-		echo "  = $s:$l:$f()"
+	for (( i=$skip; i < ${#BASH_LINENO[*]} ; i++ )) ; do
+		local src=${BASH_SOURCE[$i]}
+		local lineno=${BASH_LINENO[$i-1]}
+		local funcname=${FUNCNAME[$i]}
+		echo "  = $src:$lineno:$funcname()"
 	done
 }
 
-##################################
-# Test interface
-##################################
-
-error_noexit() {
+report_error() {
 	local TYPE=${TYPE:-"FAIL"}
 
 	local dump=true
@@ -4760,10 +4725,8 @@ error_noexit() {
 		dump=false
 	fi
 
-
 	log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ "
-	print_stack_trace >&2
-
+	(print_stack_trace 2) >&2
 	mkdir -p $LOGDIR
 	# We need to dump the logs on all nodes
 	if $dump; then
@@ -4782,6 +4745,17 @@ error_noexit() {
 			echo "$@" > $LOGDIR/err
 		fi
 	fi
+
+	# cleanup the env for failed tests
+	reset_fail_loc
+}
+
+##################################
+# Test interface
+##################################
+
+error_noexit() {
+	report_error "$@"
 }
 
 exit_status () {
@@ -4793,12 +4767,13 @@ exit_status () {
 }
 
 error() {
-	error_noexit "$@"
+	report_error "$@"
 	exit 1
 }
 
 error_exit() {
-	error "$@"
+	report_error "$@"
+	exit 1
 }
 
 # use only if we are ignoring failures for this test, bugno required.
@@ -4808,11 +4783,11 @@ error_exit() {
 error_ignore() {
 	local TYPE="IGNORE ($1)"
 	shift
-	error_noexit "$@"
+	report_error "$@"
 }
 
 error_and_remount() {
-	error_noexit "$@"
+	report_error "$@"
 	remount_client $MOUNT
 	exit 1
 }
@@ -4951,17 +4926,18 @@ log() {
 	echo "$*" >&2
 	load_module ../libcfs/libcfs/libcfs
 
-    local MSG="$*"
-    # Get rid of '
-    MSG=${MSG//\'/\\\'}
-    MSG=${MSG//\(/\\\(}
-    MSG=${MSG//\)/\\\)}
-    MSG=${MSG//\;/\\\;}
-    MSG=${MSG//\|/\\\|}
-    MSG=${MSG//\>/\\\>}
-    MSG=${MSG//\</\\\<}
-    MSG=${MSG//\//\\\/}
-    do_nodes $(comma_list $(nodes_list)) $LCTL mark "$MSG" 2> /dev/null || true
+	local MSG="$HOSTNAME: $*"
+	# Get rid of '
+	MSG=${MSG//\'/\\\'}
+	MSG=${MSG//\(/\\\(}
+	MSG=${MSG//\)/\\\)}
+	MSG=${MSG//\;/\\\;}
+	MSG=${MSG//\|/\\\|}
+	MSG=${MSG//\>/\\\>}
+	MSG=${MSG//\</\\\<}
+	MSG=${MSG//\//\\\/}
+	do_nodes $(comma_list $(nodes_list)) $LCTL mark "$MSG" 2> /dev/null ||
+		true
 }
 
 trace() {
@@ -5024,6 +5000,18 @@ banner() {
     log "$msg== $(date +"%H:%M:%S (%s)")"
 }
 
+check_dmesg_for_errors() {
+	local res
+	local errors="VFS: Busy inodes after unmount of\|\
+ldiskfs_check_descriptors: Checksum for group 0 failed\|\
+group descriptors corrupted"
+
+	res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors")
+	[ -z "$res" ] && return 0
+	echo "Kernel error detected: $res"
+	return 1
+}
+
 #
 # Run a single test function and cleanup after it.
 #
@@ -5039,12 +5027,17 @@ run_one() {
 	local SAVE_UMASK=`umask`
 	umask 0022
 
+	if ! grep -q $DIR /proc/mounts; then
+		$SETUP
+	fi
+
 	banner "test $testnum: $message"
 	test_${testnum} || error "test_$testnum failed with $?"
 	cd $SAVE_PWD
 	reset_fail_loc
 	check_grant ${testnum} || error "check_grant $testnum failed with $?"
 	check_catastrophe || error "LBUG/LASSERT detected"
+	check_dmesg_for_errors || error "Error in dmesg detected"
 	if [ "$PARALLEL" != "yes" ]; then
 		ps auxww | grep -v grep | grep -q multiop &&
 					error "multiop still running"
@@ -5053,6 +5046,7 @@ run_one() {
 	unset tdir
 	unset tfile
 	umask $SAVE_UMASK
+	$CLEANUP
 	return 0
 }
 
@@ -5451,12 +5445,7 @@ get_random_entry () {
 }
 
 client_only () {
-    [ "$CLIENTONLY" ] || [ "$CLIENTMODSONLY" = yes ]
-}
-
-is_patchless ()
-{
-    lctl get_param version | grep -q patchless
+	[ "$CLIENTONLY" ] || [ "$CLIENTMODSONLY" = yes ]
 }
 
 check_versions () {
@@ -6165,19 +6154,21 @@ create_pool() {
 }
 
 add_pool_to_list () {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
+	local fsname=${1%%.*}
+	local poolname=${1##$fsname.}
 
-    local listvar=${fsname}_CREATED_POOLS
-    eval export ${listvar}=$(expand_list ${!listvar} $poolname)
+	local listvar=${fsname}_CREATED_POOLS
+	local temp=${listvar}=$(expand_list ${!listvar} $poolname)
+	eval export $temp
 }
 
 remove_pool_from_list () {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
+	local fsname=${1%%.*}
+	local poolname=${1##$fsname.}
 
-    local listvar=${fsname}_CREATED_POOLS
-    eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+	local listvar=${fsname}_CREATED_POOLS
+	local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+	eval export $temp
 }
 
 destroy_pool_int() {
@@ -6314,6 +6305,31 @@ max_recovery_time() {
 	echo -n $service_time
 }
 
+recovery_time_min() {
+	local connection_switch_min=5
+	local connection_switch_inc=5
+	local connection_switch_max
+	local reconnect_delay_max
+	local initial_connect_timeout
+	local max
+	local timout_20
+
+	#connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT)
+	(($connection_switch_min > $TIMEOUT)) &&
+		max=$connection_switch_min || max=$TIMEOUT
+	(($max < 50)) && connection_switch_max=$max || connection_switch_max=50
+
+	#initial_connect_timeout = max(connection_switch_min, obd_timeout/20)
+	timeout_20=$((TIMEOUT/20))
+	(($connection_switch_min > $timeout_20)) &&
+		initial_connect_timeout=$connection_switch_min ||
+		initial_connect_timeout=$timeout_20
+
+	reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \
+			       initial_connect_timeout))
+	echo $((2 * reconnect_delay_max))
+}
+
 get_clients_mount_count () {
     local clients=${CLIENTS:-`hostname`}