X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=57274557985e986f884bf17a08cc0565e8258d97;hb=10ec9eba801dc80f1ccb9f8fbcbd4b0258940623;hp=401a806991f09db3806ad20eedb5c403d23aa211;hpb=183f0a19969ce1fc10b8bf5156beaa1fc29bbd0b;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 401a806..5727455 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -581,14 +581,18 @@ load_modules_local() {
 
 	set_default_debug
 	load_module ../lnet/lnet/lnet
-	case $NETTYPE in
-	o2ib)
-		LNETLND="o2iblnd/ko2iblnd"
-		;;
-	*)
-		;;
-	esac
-    LNETLND=${LNETLND:-"socklnd/ksocklnd"}
+
+	LNDPATH=${LNDPATH:-"../lnet/klnds"}
+	if [ -z "$LNETLND" ]; then
+		case $NETTYPE in
+		o2ib*)  LNETLND="o2iblnd/ko2iblnd" ;;
+		tcp*)   LNETLND="socklnd/ksocklnd" ;;
+		*)      local lnd="${NETTYPE%%[0-9]}lnd"
+			[ -f "$LNDPATH/$lnd/k$lnd.ko" ] &&
+				LNETLND="$lnd/k$lnd" ||
+				LNETLND="socklnd/ksocklnd"
+		esac
+	fi
     load_module ../lnet/klnds/$LNETLND
     load_module obdclass/obdclass
     load_module ptlrpc/ptlrpc
@@ -611,7 +615,7 @@ load_modules_local() {
 		[ "$LQUOTA" != "no" ] &&
 			load_module quota/lquota $LQUOTAOPTS
 		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
-			modprobe zfs
+			lsmod | grep zfs >&/dev/null || modprobe zfs
 			load_module osd-zfs/osd_zfs
 		fi
 		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
@@ -1092,7 +1096,7 @@ create_zpool() {
 	shift 3
 	local opts=${@:-"-o cachefile=none"}
 
-	do_facet $facet "modprobe zfs;
+	do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs;
 		$ZPOOL list -H $poolname >/dev/null 2>&1 ||
 		$ZPOOL create -f $opts $poolname $vdev"
 }
@@ -1159,7 +1163,7 @@ import_zpool() {
 
 	if [[ -n "$poolname" ]]; then
 		opts+=" -d $(dirname $(facet_vdevice $facet))"
-		do_facet $facet "modprobe zfs;
+		do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs;
 			$ZPOOL list -H $poolname >/dev/null 2>&1 ||
 			$ZPOOL import -f $opts $poolname"
 	fi
@@ -1404,7 +1408,7 @@ mount_facet() {
 	# commit the device label change to disk
 	if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then
 		echo "Commit the device label on ${!dev}"
-		do_facet $facet "sync; sleep 5; sync"
+		do_facet $facet "sync; sleep 1; sync"
 	fi
 
 
@@ -1772,14 +1776,24 @@ zconf_mount_clients() {
 	fi
 
 	do_nodes $clients "
-running=\\\$(mount | grep -c $mnt' ');
-rc=0;
-if [ \\\$running -eq 0 ] ; then
-    mkdir -p $mnt;
-    $MOUNT_CMD $flags $opts $device $mnt;
-    rc=\\\$?;
-fi;
-exit \\\$rc" || return ${PIPESTATUS[0]}
+		running=\\\$(mount | grep -c $mnt' ');
+		rc=0;
+		if [ \\\$running -eq 0 ] ; then
+			mkdir -p $mnt;
+			$MOUNT_CMD $flags $opts $device $mnt;
+			rc=\\\$?;
+		else
+			lustre_mnt_count=\\\$(mount | grep $mnt' ' | \
+				grep 'type lustre' | wc -l);
+			if [ \\\$running -ne \\\$lustre_mnt_count ] ; then
+				echo zconf_mount_clients FAILED: \
+					mount count \\\$running, not matching \
+					with mount count of 'type lustre' \
+					\\\$lustre_mnt_count;
+				rc=1;
+			fi;
+		fi;
+	exit \\\$rc" || return ${PIPESTATUS[0]}
 
 	echo "Started clients $clients: "
 	do_nodes $clients "mount | grep $mnt' '"
@@ -1960,29 +1974,31 @@ node_var_name() {
 }
 
 start_client_load() {
-    local client=$1
-    local load=$2
-    local var=$(node_var_name $client)_load
-    eval export ${var}=$load
-
-    do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
-BREAK_ON_ERROR=$BREAK_ON_ERROR \
-END_RUN_FILE=$END_RUN_FILE \
-LOAD_PID_FILE=$LOAD_PID_FILE \
-TESTLOG_PREFIX=$TESTLOG_PREFIX \
-TESTNAME=$TESTNAME \
-DBENCH_LIB=$DBENCH_LIB \
-DBENCH_SRC=$DBENCH_SRC \
-CLIENT_COUNT=$((CLIENTCOUNT - 1)) \
-LFS=$LFS \
-run_${load}.sh" &
-    local ppid=$!
-    log "Started client load: ${load} on $client"
-
-    # get the children process IDs
-    local pids=$(ps --ppid $ppid -o pid= | xargs)
-    CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids"
-    return 0
+	local client=$1
+	local load=$2
+	local var=$(node_var_name $client)_load
+	eval export ${var}=$load
+
+	do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
+			BREAK_ON_ERROR=$BREAK_ON_ERROR \
+			END_RUN_FILE=$END_RUN_FILE \
+			LOAD_PID_FILE=$LOAD_PID_FILE \
+			TESTLOG_PREFIX=$TESTLOG_PREFIX \
+			TESTNAME=$TESTNAME \
+			DBENCH_LIB=$DBENCH_LIB \
+			DBENCH_SRC=$DBENCH_SRC \
+			CLIENT_COUNT=$((CLIENTCOUNT - 1)) \
+			LFS=$LFS \
+			LCTL=$LCTL \
+			FSNAME=$FSNAME \
+			run_${load}.sh" &
+	local ppid=$!
+	log "Started client load: ${load} on $client"
+
+	# get the children process IDs
+	local pids=$(ps --ppid $ppid -o pid= | xargs)
+	CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids"
+	return 0
 }
 
 start_client_loads () {
@@ -2424,10 +2440,10 @@ wait_osts_up() {
 	wait_update $HOSTNAME "eval $cmd" $OSTCOUNT ||
 		error "wait_update OSTs up on client failed"
 
-	cmd="$LCTL get_param -n lod.$FSNAME-MDT*-*.target_obd | sort -u |
-	     awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'"
+	cmd="$LCTL get_param osp.$FSNAME-OST*-MDT0000.prealloc_last_id |
+	     awk '/=[1-9][0-9]/ { c += 1 } END { printf \\\"%d\\\", c }'"
 	wait_update_facet $SINGLEMDS "eval $cmd" $OSTCOUNT ||
-		error "wait_update OSTs up on MDT failed"
+		error "wait_update OSTs up on MDT0000 failed"
 }
 
 wait_destroy_complete () {
@@ -2751,21 +2767,11 @@ fail_abort() {
 	clients_up || error "post-failover stat: $?"
 }
 
-do_lmc() {
-    echo There is no lmc.  This is mountconf, baby.
-    exit 1
-}
-
 host_nids_address() {
-    local nodes=$1
-    local kind=$2
+	local nodes=$1
+	local net=${2:-"."}
 
-    if [ -n "$kind" ]; then
-        nids=$(do_nodes $nodes "$LCTL list_nids | grep $kind | cut -f 1 -d '@'")
-    else
-        nids=$(do_nodes $nodes "$LCTL list_nids all | cut -f 1 -d '@'")
-    fi
-    echo $nids
+	do_nodes $nodes "$LCTL list_nids | grep $net | cut -f 1 -d @"
 }
 
 h2name_or_ip() {
@@ -2774,40 +2780,34 @@ h2name_or_ip() {
 	fi
 }
 
-h2ptl() {
-	if [ "$1" = "'*'" ]; then echo \'*\'; else
-		ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | \
-							awk '{print $1}'`
-		if [ -z "$ID" ]; then
-			echo "Could not get a ptl id for $1..."
-			exit 1
-		fi
-		echo $ID"@ptl"
+h2nettype() {
+	if [[ -n "$NETTYPE" ]]; then
+		h2name_or_ip "$1" "$NETTYPE"
+	else
+		h2name_or_ip "$1" "$2"
 	fi
 }
-declare -fx h2ptl
+declare -fx h2nettype
 
+# Wrapper function to print the deprecation warning
 h2tcp() {
-	h2name_or_ip "$1" "tcp"
-}
-declare -fx h2tcp
-
-h2elan() {
-	if [ "$1" = "'*'" ]; then echo \'*\'; else
-		if type __h2elan >/dev/null 2>&1; then
-			ID=$(__h2elan $1)
-		else
-			ID=`echo $1 | sed 's/[^0-9]*//g'`
-		fi
-		echo $ID"@elan"
+	echo "h2tcp: deprecated, use h2nettype instead" 1>&2
+	if [[ -n "$NETTYPE" ]]; then
+		h2nettype "$@"
+	else
+		h2nettype "$1" "tcp"
 	fi
 }
-declare -fx h2elan
 
+# Wrapper function to print the deprecation warning
 h2o2ib() {
-	h2name_or_ip "$1" "o2ib"
+	echo "h2o2ib: deprecated, use h2nettype instead" 1>&2
+	if [[ -n "$NETTYPE" ]]; then
+		h2nettype "$@"
+	else
+		h2nettype "$1" "o2ib"
+	fi
 }
-declare -fx h2o2ib
 
 # This enables variables in cfg/"setup".sh files to support the pdsh HOSTLIST
 # expressions format. As a bonus we can then just pass in those variables
@@ -2843,6 +2843,8 @@ hostlist_expand() {
                 group=${group%%]*}
 
                 for range in ${group//,/ }; do
+		    local order
+
                     begin=${range%-*}
                     end=${range#*-}
 
@@ -2858,7 +2860,13 @@ hostlist_expand() {
                     begin=$(echo $begin | sed 's/0*//')
                     [ -z $begin ] && begin=0
 
-                    for num in $(seq -f "%0${padlen}g" $begin $end); do
+		    if [ ! -z "${begin##[!0-9]*}" ]; then
+			order=$(seq -f "%0${padlen}g" $begin $end)
+		    else
+			order=$(eval echo {$begin..$end});
+		    fi
+
+		    for num in $order; do
                         value="${name#*,}${num}${back}"
                         [ "$value" != "${value/\[/}" ] && {
                             value=$(hostlist_expand "$value")
@@ -3544,7 +3552,7 @@ mkfs_opts() {
 
 	var=${facet}failover_HOST
 	if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then
-		opts+=" --failnode=$(h2$NETTYPE ${!var})"
+		opts+=" --failnode=$(h2nettype ${!var})"
 	fi
 
 	opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"}
@@ -3609,6 +3617,17 @@ mkfs_opts() {
 	echo -n "$opts"
 }
 
+mountfs_opts() {
+	local facet=$1
+	local type=$(facet_type $facet)
+	local var=${type}_MOUNT_FS_OPTS
+	local opts=""
+	if [ -n "${!var}" ]; then
+		opts+=" --mountfsoptions=${!var}"
+	fi
+	echo -n "$opts"
+}
+
 check_ost_indices() {
 	local index_count=${#OST_INDICES[@]}
 	[[ $index_count -eq 0 || $OSTCOUNT -le $index_count ]] && return 0
@@ -3627,6 +3646,34 @@ check_ost_indices() {
 	done
 }
 
+__touch_device()
+{
+	local facet_type=$1 # mgs || mds || ost
+	local facet_num=$2
+	local facet=${1}${2}
+	local device
+
+	case "$(facet_fstype $facet)" in
+	ldiskfs)
+		device=$(${facet_type}devname $facet_num)
+		;;
+	zfs)
+		device=$(${facet_type}vdevname $facet_num)
+		;;
+	*)
+		error "Unhandled filesystem type"
+		;;
+	esac
+
+	do_facet $facet "[ -e \"$device\" ]" && return
+
+	# Note: the following check only works with absolute paths
+	[[ ! "$device" =~ ^/dev/ ]] || [[ "$device" =~ ^/dev/shm/ ]] ||
+		error "$facet: device '$device' does not exist"
+
+	do_facet $facet "touch \"${device}\""
+}
+
 format_mgs() {
 	local quiet
 
@@ -3635,7 +3682,13 @@ format_mgs() {
 	fi
 	echo "Format mgs: $(mgsdevname)"
 	reformat_external_journal mgs
-	add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+
+	# touch "device" in case it is a loopback file for testing and needs to
+	# be created. mkfs.lustre doesn't do this to avoid accidentally writing
+	# to non-existent files in /dev if the admin made a typo during setup
+	__touch_device mgs
+
+	add mgs $(mkfs_opts mgs $(mgsdevname)) $(mountfs_opts mgs) --reformat \
 		$(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
 }
 
@@ -3648,9 +3701,12 @@ format_mdt() {
 	fi
 	echo "Format mds$num: $(mdsdevname $num)"
 	reformat_external_journal mds$num
+
+	__touch_device mds $num
+
 	add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
-		--reformat $(mdsdevname $num) $(mdsvdevname $num) \
-		${quiet:+>/dev/null} || exit 10
+		$(mountfs_opts mds$num) --reformat $(mdsdevname $num) \
+		$(mdsvdevname $num) ${quiet:+>/dev/null} || exit 10
 }
 
 format_ost() {
@@ -3661,9 +3717,12 @@ format_ost() {
 	fi
 	echo "Format ost$num: $(ostdevname $num)"
 	reformat_external_journal ost$num
+
+	__touch_device ost $num
+
 	add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
-		--reformat $(ostdevname $num) $(ostvdevname ${num}) \
-		${quiet:+>/dev/null} || exit 10
+		$(mountfs_opts ost$num) --reformat $(ostdevname $num) \
+		$(ostvdevname ${num}) ${quiet:+>/dev/null} || exit 10
 }
 
 formatall() {
@@ -4073,20 +4132,10 @@ check_config_client () {
         return 0
     fi
 
-    local myMGS_host=$mgs_HOST
-    if [ "$NETTYPE" = "ptl" ]; then
-        myMGS_host=$(h2ptl $mgs_HOST | sed -e s/@ptl//)
-    fi
-
     echo Checking config lustre mounted on $mntpt
     local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}')
     mgshost=$(echo $mgshost | awk -F: '{print $1}')
 
-#    if [ "$mgshost" != "$myMGS_host" ]; then
-#            log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
-#                   Please use correct config or set mds_HOST correctly!"
-#    fi
-
 }
 
 check_config_clients () {
@@ -5293,8 +5342,9 @@ check_grant() {
 	export base=$(basetest $1)
 	[ "$CHECK_GRANT" == "no" ] && return 0
 
-	testname=GCHECK_ONLY_${base}
-	[ ${!testname}x == x ] && return 0
+	testnamebase=GCHECK_ONLY_${base}
+	testname=GCHECK_ONLY_$1
+	[ ${!testnamebase}x == x -a ${!testname}x == x ] && return 0
 
 	echo -n "checking grant......"
 
@@ -5310,17 +5360,22 @@ check_grant() {
 		awk '{ total += $1 } END { printf("%0.0f", total) }')
 
 	# get server grant
+	# which is tot_granted less grant_precreate
 	server_grant=$(do_nodes $(comma_list $(osts_nodes)) \
-		"$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" |
-		awk '{ total += $1 } END { printf("%0.0f", total) }')
+		"$LCTL get_param "\
+		"obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" |
+		sed 's/=/ /'| awk '/tot_granted/{ total += $2 }; 
+				/tot_pending/{ total -= $2 };
+				/grant_precreate/{ total -= $2 };
+				END { printf("%0.0f", total) }')
 
 	# check whether client grant == server grant
 	if [[ $client_grant -ne $server_grant ]]; then
-		echo "failed: client:${client_grant} server: ${server_grant}."
 		do_nodes $(comma_list $(osts_nodes)) \
-			"$LCTL get_param obdfilter.${FSNAME}-OST*.tot*"
+			"$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \
+		        "obdfilter.${FSNAME}-OST*.grant_*"
 		do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
-		return 1
+		error "failed: client:${client_grant} server: ${server_grant}."
 	else
 		echo "pass: client:${client_grant} server: ${server_grant}"
 	fi
@@ -5662,13 +5717,19 @@ get_stripe () {
 
 setstripe_nfsserver () {
 	local dir=$1
+	local nfsexportdir=$2
+	shift
+	shift
 
-	local nfsserver=$(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \
-		{ print $1 }' /proc/mounts | cut -f 1 -d : | head -n1)
+	local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \
+		{ print $1 }' /proc/mounts | cut -f 1 -d :))
 
-	[ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1
+	# check that only one nfs mounted
+	[[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1
+	(( ${#nfsexport[@]} == 1 )) ||
+		error "several nfs mounts found for $dir: ${nfsexport[@]} !"
 
-	do_nodev $nfsserver lfs setstripe "$@"
+	do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@"
 }
 
 # Check and add a test group.
@@ -5838,32 +5899,22 @@ inodes_available () {
 }
 
 mdsrate_inodes_available () {
-    local min_inodes=$(inodes_available)
-    echo $((min_inodes * 99 / 100))
-}
-
-# reset llite stat counters
-clear_llite_stats(){
-        lctl set_param -n llite.*.stats 0
-}
-
-# sum llite stat items
-calc_llite_stats() {
-	local res=$(lctl get_param -n llite.*.stats |
-		awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }')
-	echo $((res))
+	local min_inodes=$(inodes_available)
+	echo $((min_inodes * 99 / 100))
 }
 
-# reset osc stat counters
-clear_osc_stats(){
-	lctl set_param -n osc.*.osc_stats 0
+# reset stat counters
+clear_stats() {
+	local paramfile="$1"
+	lctl set_param -n $paramfile=0
 }
 
-# sum osc stat items
-calc_osc_stats() {
-	local res=$(lctl get_param -n osc.*.osc_stats |
-		awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }')
-	echo $((res))
+# sum stat items
+calc_stats() {
+	local paramfile="$1"
+	local stat="$2"
+	lctl get_param -n $paramfile |
+		awk '/^'$stat'/ { sum += $2 } END { printf("%0.0f", sum) }'
 }
 
 calc_sum () {
@@ -5871,8 +5922,8 @@ calc_sum () {
 }
 
 calc_osc_kbytes () {
-        df $MOUNT > /dev/null
-        $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
+	df $MOUNT > /dev/null
+	$LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
 }
 
 # save_lustre_params(comma separated facet list, parameter_mask)
@@ -6111,7 +6162,6 @@ _wait_osc_import_state() {
 	if [[ $facet == client* ]]; then
 		# During setup time, the osc might not be setup, it need wait
 		# until list_param can return valid value.
-		param="osc.${ost%?}[^mM]*.ost_server_uuid"
 		params=$($LCTL list_param $param 2>/dev/null || true)
 		while [ -z "$params" ]; do
 			if [ $i -ge $maxtime ]; then
@@ -7097,7 +7147,7 @@ is_sanity_benchmark() {
 }
 
 min_ost_size () {
-    $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1
+	$LFS df | grep OST | awk '{print $4}' | sort -un | head -1
 }
 
 #