X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=lustre%2Ftests%2Ftest-framework.sh;h=3d0398f19679e234da3ea0b21a290a9c71302659;hb=af666bef058c5b7997527fc851a84a89375912fb;hp=e337b9d9d348278687d0608b95294210a6059235;hpb=436cd4fd21ffee5830c9b4e75055db80c47547d5;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index e337b9d..3d0398f 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -299,6 +299,9 @@ init_test_env() {
 	[ ! -f "$LCTL" ] && export LCTL=$(which lctl)
 	export LFS=${LFS:-"$LUSTRE/utils/lfs"}
 	[ ! -f "$LFS" ] && export LFS=$(which lfs)
+	export KSOCKLND_CONFIG=${KSOCKLND_CONFIG:-"$LUSTRE/scripts/ksocklnd-config"}
+	[ ! -f "$KSOCKLND_CONFIG" ] &&
+		export KSOCKLND_CONFIG=$(which ksocklnd-config 2> /dev/null)
 
 	export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"}
 
@@ -1566,8 +1569,7 @@ get_osd_param() {
 	local device=${2:-$FSNAME-OST*}
 	local name=$3
 
-	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
-		osd-*.$device.$name 2>&1" | grep -v 'error:'
+	do_nodes $nodes "$LCTL get_param -n osd-*.$device.$name"
 }
 
 set_osd_param() {
@@ -1576,8 +1578,7 @@ set_osd_param() {
 	local name=$3
 	local value=$4
 
-	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
-		osd-*.$device.$name=$value 2>&1" | grep -v 'error:'
+	do_nodes $nodes "$LCTL set_param -n osd-*.$device.$name=$value"
 }
 
 set_debug_size () {
@@ -2727,11 +2728,12 @@ remount_facet() {
 reboot_facet() {
 	local facet=$1
 	local node=$(facet_active_host $facet)
+	local sleep_time=${2:-10}
 
 	if [ "$FAILURE_MODE" = HARD ]; then
 		boot_node $node
 	else
-		sleep 10
+		sleep $sleep_time
 	fi
 }
 
@@ -3661,7 +3663,7 @@ facet_failover() {
 		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
 		echo reboot facets: ${affecteds[index]}
 
-		reboot_facet $facet
+		reboot_facet $facet $sleep_time
 
 		change_active ${affecteds[index]}
 
@@ -5840,30 +5842,30 @@ check_and_cleanup_lustre() {
 # General functions
 
 wait_for_function () {
-    local quiet=""
-
-    # suppress fn both stderr and stdout
-    if [ "$1" = "--quiet" ]; then
-        shift
-        quiet=" > /dev/null 2>&1"
+	local quiet=""
 
-    fi
+	# suppress fn both stderr and stdout
+	if [ "$1" = "--quiet" ]; then
+		shift
+		quiet=" > /dev/null 2>&1"
+	fi
 
-    local fn=$1
-    local max=${2:-900}
-    local sleep=${3:-5}
+	local fn=$1
+	local max=${2:-900}
+	local sleep=${3:-5}
 
-    local wait=0
+	local wait=0
 
-    while true; do
+	while true; do
 
-        eval $fn $quiet && return 0
+		eval $fn $quiet && return 0
 
-        wait=$((wait + sleep))
-        [ $wait -lt $max ] || return 1
-        echo waiting $fn, $((max - wait)) secs left ...
-        sleep $sleep
-    done
+		[ $wait -lt $max ] || return 1
+		echo waiting $fn, $((max - wait)) secs left ...
+		wait=$((wait + sleep))
+		[ $wait -gt $max ] && ((sleep -= wait - max))
+		sleep $sleep
+	done
 }
 
 check_network() {
@@ -6820,7 +6822,7 @@ ostuuid_from_index()
 }
 
 ostname_from_index() {
-    local uuid=$(ostuuid_from_index $1)
+    local uuid=$(ostuuid_from_index $1 $2)
     echo ${uuid/_UUID/}
 }
 
@@ -7454,48 +7456,49 @@ get_osc_import_name() {
 }
 
 _wait_import_state () {
-    local expected=$1
-    local CONN_PROC=$2
-    local maxtime=${3:-$(max_recovery_time)}
-    local error_on_failure=${4:-1}
-    local CONN_STATE
-    local i=0
+	local expected="$1"
+	local CONN_PROC="$2"
+	local maxtime=${3:-$(max_recovery_time)}
+	local err_on_fail=${4:-1}
+	local CONN_STATE
+	local i=0
 
 	CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq)
-    while ! echo "${CONN_STATE}" | egrep -q "^${expected}\$" ; do
-        if [ "${expected}" == "DISCONN" ]; then
-            # for disconn we can check after proc entry is removed
-            [ "x${CONN_STATE}" == "x" ] && return 0
-            #  with AT enabled, we can have connect request timeout near of
-            # reconnect timeout and test can't see real disconnect
-            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
-        fi
-	if [ $i -ge $maxtime ]; then
-	    [ $error_on_failure -ne 0 ] && \
-		error "can't put import for $CONN_PROC into ${expected}" \
-		      "state after $i sec, have ${CONN_STATE}"
-            return 1
-	fi
-        sleep 1
-	# Add uniq for multi-mount case
-	CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq)
-        i=$(($i + 1))
-    done
+	while ! echo "${CONN_STATE}" | egrep -q "^${expected}\$" ; do
+		if [[ "${expected}" == "DISCONN" ]]; then
+			# for disconn we can check after proc entry is removed
+			[[ -z "${CONN_STATE}" ]] && return 0
+			# with AT, we can have connect request timeout near
+			# reconnect timeout and test can't see real disconnect
+			[[ "${CONN_STATE}" == "CONNECTING" ]] && return 0
+		fi
+		if (( $i >= $maxtime )); then
+			(( $err_on_fail != 0 )) &&
+				error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}"
+			return 1
+		fi
+		sleep 1
+		# Add uniq for multi-mount case
+		CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null |
+			     cut -f2 | uniq)
+		i=$((i + 1))
+	done
 
-    log "$CONN_PROC in ${CONN_STATE} state after $i sec"
-    return 0
+	log "$CONN_PROC in ${CONN_STATE} state after $i sec"
+	return 0
 }
 
 wait_import_state() {
-    local state=$1
-    local params=$2
-    local maxtime=${3:-$(max_recovery_time)}
-    local error_on_failure=${4:-1}
-    local param
-
-    for param in ${params//,/ }; do
-	_wait_import_state $state $param $maxtime $error_on_failure || return
-    done
+	local expected="$1"
+	local params="$2"
+	local maxtime=${3:-$(max_recovery_time)}
+	local err_on_fail=${4:-1}
+	local param
+
+	for param in ${params//,/ }; do
+		_wait_import_state "$expected" "$param" $maxtime $err_on_fail ||
+		return
+	done
 }
 
 wait_import_state_mount() {
@@ -7503,7 +7506,7 @@ wait_import_state_mount() {
 		return 0
 	fi
 
-	wait_import_state $*
+	wait_import_state "$@"
 }
 
 # One client request could be timed out because server was not ready
@@ -7702,11 +7705,10 @@ do_rpc_nodes () {
 }
 
 wait_clients_import_state () {
-	local list=$1
-	local facet=$2
-	local expected=$3
-
-	local facets=$facet
+	local list="$1"
+	local facet="$2"
+	local expected="$3"
+	local facets="$facet"
 
 	if [ "$FAILURE_MODE" = HARD ]; then
 		facets=$(facets_on_host $(facet_active_host $facet))
@@ -7717,11 +7719,11 @@ wait_clients_import_state () {
 		local proc_path
 		case $facet in
 		ost* ) proc_path="osc.$(get_clientosc_proc_path \
-				  $label).ost_server_uuid" ;;
+					$label).ost_server_uuid" ;;
 		mds* ) proc_path="mdc.$(get_clientmdc_proc_path \
-				  $label).mds_server_uuid" ;;
+					$label).mds_server_uuid" ;;
 		mgs* ) proc_path="mgc.$(get_clientmgc_proc_path \
-				  $label).mgs_server_uuid" ;;
+					$label).mgs_server_uuid" ;;
 		*) error "unknown facet!" ;;
 		esac
 
@@ -10676,3 +10678,59 @@ wait_nm_sync() {
 	fi
 	echo "waited $((i - 1)) seconds for sync"
 }
+
+consume_precreations() {
+	local dir=$1
+	local mfacet=$2
+	local OSTIDX=$3
+	local extra=${4:-2}
+	local OST=$(ostname_from_index $OSTIDX $dir)
+
+	test_mkdir -p $dir/${OST}
+	$LFS setstripe -i $OSTIDX -c 1 ${dir}/${OST}
+
+	# on the mdt's osc
+	local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST)
+	local last_id=$(do_facet $mfacet $LCTL get_param -n \
+			osp.$mdtosc_proc.prealloc_last_id)
+	local next_id=$(do_facet $mfacet $LCTL get_param -n \
+			osp.$mdtosc_proc.prealloc_next_id)
+	echo "Creating to objid $last_id on ost $OST..."
+	createmany -o $dir/${OST}/f $next_id $((last_id - next_id + extra))
+}
+
+__exhaust_precreations() {
+	local OSTIDX=$1
+	local FAILLOC=$2
+	local FAILIDX=${3:-$OSTIDX}
+	local ofacet=ost$((OSTIDX + 1))
+
+	mkdir_on_mdt0 $DIR/$tdir
+	local mdtidx=$($LFS getstripe -m $DIR/$tdir)
+	local mfacet=mds$((mdtidx + 1))
+	echo OSTIDX=$OSTIDX MDTIDX=$mdtidx
+
+	local mdtosc_proc=$(get_mdtosc_proc_path $mfacet)
+	do_facet $mfacet $LCTL get_param osp.$mdtosc_proc.prealloc*
+
+#define OBD_FAIL_OST_ENOSPC              0x215
+	do_facet $ofacet $LCTL set_param fail_val=$FAILIDX fail_loc=0x215
+
+	consume_precreations $DIR/$tdir $mfacet $OSTIDX
+
+	do_facet $mfacet $LCTL get_param osp.$mdtosc_proc.prealloc*
+	do_facet $ofacet $LCTL set_param fail_loc=$FAILLOC
+}
+
+exhaust_precreations() {
+	__exhaust_precreations $1 $2 $3
+	sleep_maxage
+}
+
+exhaust_all_precreations() {
+	local i
+	for (( i=0; i < OSTCOUNT; i++ )) ; do
+		__exhaust_precreations $i $1 -1
+	done
+	sleep_maxage
+}