X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=fa3dadbe5c6031c6f6cd51de2ded0b94161083e6;hb=a9257f57ad27a1b1219cfd8ce29fbfabdca1df84;hp=c1e9a9c79caf6dc92d200b8fc4bae022ff8eecba;hpb=189c466ff8f7cbf6f1ebb53295f09df2ae8237a2;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index c1e9a9c..fa3dadb 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -123,14 +123,61 @@ init_test_env() {
     export TEST_FAILED=false
     export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false}
 
-    export MKE2FS=${MKE2FS:-mke2fs}
-    export DEBUGFS=${DEBUGFS:-debugfs}
-    export TUNE2FS=${TUNE2FS:-tune2fs}
-    export E2LABEL=${E2LABEL:-e2label}
-    export DUMPE2FS=${DUMPE2FS:-dumpe2fs}
-    export E2FSCK=${E2FSCK:-e2fsck}
-    export LFSCK_BIN=${LFSCK_BIN:-lfsck}
+    export MKE2FS=$MKE2FS
+    if [ -z "$MKE2FS" ]; then
+        if which mkfs.ldiskfs >/dev/null 2>&1; then
+            export MKE2FS=mkfs.ldiskfs
+        else
+            export MKE2FS=mke2fs
+        fi
+    fi
+
+    export DEBUGFS=$DEBUGFS
+    if [ -z "$DEBUGFS" ]; then
+        if which debugfs.ldiskfs >/dev/null 2>&1; then
+            export DEBUGFS=debugfs.ldiskfs
+        else
+            export DEBUGFS=debugfs
+        fi
+    fi
 
+    export TUNE2FS=$TUNE2FS
+    if [ -z "$TUNE2FS" ]; then
+        if which tunefs.ldiskfs >/dev/null 2>&1; then
+            export TUNE2FS=tunefs.ldiskfs
+        else
+            export TUNE2FS=tune2fs
+        fi
+    fi
+
+    export E2LABEL=$E2LABEL
+    if [ -z "$E2LABEL" ]; then
+        if which label.ldiskfs >/dev/null 2>&1; then
+            export E2LABEL=label.ldiskfs
+        else
+            export E2LABEL=e2label
+        fi
+    fi
+
+    export DUMPE2FS=$DUMPE2FS
+    if [ -z "$DUMPE2FS" ]; then
+        if which dumpfs.ldiskfs >/dev/null 2>&1; then
+            export DUMPE2FS=dumpfs.ldiskfs
+        else
+            export DUMPE2FS=dumpe2fs
+        fi
+    fi
+
+    export E2FSCK=$E2FSCK
+    if [ -z "$E2FSCK" ]; then
+        if which fsck.ldiskfs >/dev/null 2>&1; then
+            export E2FSCK=fsck.ldiskfs
+        else
+            export E2FSCK=e2fsck
+        fi
+    fi
+
+    export LFSCK_BIN=${LFSCK_BIN:-lfsck}
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
@@ -190,11 +237,11 @@ init_test_env() {
         fi
     fi
     export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"}
-    [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID=$(which ll_decode_filter_fid)
+    [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid"
     export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
-    [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre)
+    [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre"
     export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"}
-    [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre)
+    [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre"
     export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
     export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
     [ ! -f "$LUSTRE_RMMOD" ] &&
@@ -375,6 +422,7 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
+	[ $(facet_fstype ost1) == "zfs" ] && export USE_OFD=yes
 	if [ "$USE_OFD" == yes ]; then
 		if module_loaded obdfilter; then
 			if ! $LUSTRE_RMMOD ldiskfs; then
@@ -415,7 +463,6 @@ load_modules_local() {
 		fi
 	fi
 
-	local ncpts=0
 	# if there is only one CPU core, libcfs can only create one partition
 	# if there is more than 4 CPU cores, libcfs should create multiple CPU
 	# partitions. So we just force libcfs to create 2 partitions for
@@ -423,12 +470,12 @@ load_modules_local() {
 	if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then
 		# force to enable multiple CPU partitions
 		echo "Force libcfs to create 2 CPU partitions"
-		ncpts=2
+		MODOPTS_LIBCFS="cpu_npartitions=2 $MODOPTS_LIBCFS"
 	else
 		echo "libcfs will create CPU partition based on online CPUs"
 	fi
 
-	load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts
+	load_module ../libcfs/libcfs/libcfs
 
     [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG"
     [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }"
@@ -454,6 +501,7 @@ load_modules_local() {
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
 		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
 		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
+			modprobe zfs
 			load_module osd-zfs/osd_zfs
 		fi
 		load_module mgs/mgs
@@ -471,14 +519,15 @@ load_modules_local() {
 			load_module osd-ldiskfs/osd_ldiskfs
 		fi
 		load_module mdt/mdt
-		load_module cmm/cmm
 		load_module ost/ost
 		load_module lod/lod
+		load_module osp/osp
 		if [ "$USE_OFD" == yes ]; then
 			load_module ofd/ofd
 		else
 			load_module obdfilter/obdfilter
 		fi
+		load_module osp/osp
     fi
 
 
@@ -1739,23 +1788,52 @@ sync_all_data() {
 		grep -v 'Found no match'
 }
 
-wait_delete_completed () {
-    local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \
-                     awk 'BEGIN{total=0}; {total+=$1}; END{print total}'`
+wait_delete_completed_mds() {
+	local MAX_WAIT=${1:-20}
+	local mds2sync=""
+	local stime=`date +%s`
+	local etime
+	local node
+	local changes
+
+	# find MDS with pending deletions
+	for node in $(mdts_nodes); do
+		changes=$(do_node $node "lctl get_param -n osc.*MDT*.sync_*" \
+			2>/dev/null | calc_sum)
+		if [ -z "$changes" ] || [ $changes -eq 0 ]; then
+			continue
+		fi
+		mds2sync="$mds2sync $node"
+	done
+	if [ "$mds2sync" == "" ]; then
+		return
+	fi
+	mds2sync=$(comma_list $mds2sync)
+
+	# sync MDS transactions
+	do_nodes $mds2sync "lctl set_param -n osd*.*MD*.force_sync 1"
+
+	# wait till all changes are sent and commmitted by OSTs
+	# for ldiskfs space is released upon execution, but DMU
+	# do this upon commit
+
+	local WAIT=0
+	while [ "$WAIT" -ne "$MAX_WAIT" ]; do
+		changes=$(do_nodes $mds2sync "lctl get_param -n osc.*MDT*.sync_*" \
+			| calc_sum)
+		#echo "$node: $changes changes on all"
+		if [ "$changes" -eq "0" ]; then
+			etime=`date +%s`
+			#echo "delete took $((etime - stime)) seconds"
+			return
+		fi
+		sleep 1
+		WAIT=$(( WAIT + 1))
+	done
 
-    local WAIT=0
-    local MAX_WAIT=20
-    while [ "$WAIT" -ne "$MAX_WAIT" ]; do
-        sleep 1
-        TOTAL=`lctl get_param -n osc.*.kbytesavail | \
-               awk 'BEGIN{total=0}; {total+=$1}; END{print total}'`
-        [ "$TOTAL" -eq "$TOTALPREV" ] && return 0
-        echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL "
-        TOTALPREV=$TOTAL
-        WAIT=$(( WAIT + 1))
-    done
-    echo "Delete is not completed in $MAX_WAIT sec"
-    return 1
+	etime=`date +%s`
+	echo "Delete is not completed in $((etime - stime)) seconds"
+	do_nodes $mds2sync "lctl get_param osc.*MDT*.sync_*"
 }
 
 wait_for_host() {
@@ -1849,28 +1927,35 @@ wait_mds_ost_sync () {
 }
 
 wait_destroy_complete () {
-    echo "Waiting for destroy to be done..."
-    # MAX value shouldn't be big as this mean server responsiveness
-    # never increase this just to make test pass but investigate
-    # why it takes so long time
-    local MAX=5
-    local WAIT=0
-    while [ $WAIT -lt $MAX ]; do
-        local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
-        local con=1
-        for ((i=0; i<${#RPCs[@]}; i++)); do
-            [ ${RPCs[$i]} -eq 0 ] && continue
-            # there are still some destroy RPCs in flight
-            con=0
-            break;
-        done
-        sleep 1
-        [ ${con} -eq 1 ] && return 0 # done waiting
-        echo "Waiting $WAIT secs for destroys to be done."
-        WAIT=$((WAIT + 1))
-    done
-    echo "Destroys weren't done in $MAX sec."
-    return 1
+	echo "Waiting for local destroys to complete"
+	# MAX value shouldn't be big as this mean server responsiveness
+	# never increase this just to make test pass but investigate
+	# why it takes so long time
+	local MAX=5
+	local WAIT=0
+	while [ $WAIT -lt $MAX ]; do
+		local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
+		local con=1
+		local i
+
+		for ((i=0; i<${#RPCs[@]}; i++)); do
+			[ ${RPCs[$i]} -eq 0 ] && continue
+			# there are still some destroy RPCs in flight
+			con=0
+			break;
+		done
+		sleep 1
+		[ ${con} -eq 1 ] && return 0 # done waiting
+		echo "Waiting ${WAIT}s for local destroys to complete"
+		WAIT=$((WAIT + 1))
+	done
+	echo "Local destroys weren't done in $MAX sec."
+	return 1
+}
+
+wait_delete_completed() {
+	wait_delete_completed_mds $1 || return $?
+	wait_destroy_complete
 }
 
 wait_exit_ST () {
@@ -2016,7 +2101,9 @@ replay_barrier() {
 
     # make sure there will be no seq change
     local clients=${CLIENTS:-$HOSTNAME}
-    do_nodes $clients "f=${MOUNT}/fsa-\\\$(hostname); mcreate \\\$f; rm \\\$f"
+    local f=fsa-\\\$\(hostname\)
+    do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
+    do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
 
     local svc=${facet}_svc
     do_facet $facet $LCTL --device %${!svc} notransno
@@ -3013,6 +3100,25 @@ osc_ensure_active () {
     [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up"
 }
 
+set_conf_param_and_check() {
+	local myfacet=$1
+	local TEST=$2
+	local PARAM=$3
+	local ORIG=$(do_facet $myfacet "$TEST")
+	if [ $# -gt 3 ]; then
+		local FINAL=$4
+	else
+		local -i FINAL
+		FINAL=$((ORIG + 5))
+	fi
+	echo "Setting $PARAM from $ORIG to $FINAL"
+	do_facet mgs "$LCTL conf_param $PARAM='$FINAL'" ||
+		error "conf_param $PARAM failed"
+
+	wait_update $(facet_host $myfacet) "$TEST" "$FINAL" ||
+		error "check $PARAM failed!"
+}
+
 init_param_vars () {
 	remote_mds_nodsh ||
 		TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")