X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=b7216d25095334581485b72d22503749c2885c31;hp=a7c660a2f4cb6de6f5ead2a73da3dfd937558f8d;hb=50bbf65fec92adbcedf1706eb881e2a156874d0d;hpb=938947f037ee65bb695fcc4509817463842e57fa

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index a7c660a..b7216d2 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -445,19 +445,26 @@ load_modules_local() {
         grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
         grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
-		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
-			grep -q exportfs_decode_fh $SYMLIST ||
-			{ modprobe exportfs 2> /dev/null || true; }
-			load_module ../ldiskfs/ldiskfs/ldiskfs
-		fi
 		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
+			load_module osd-zfs/osd_zfs
+		fi
 		load_module mgs/mgs
 		load_module mds/mds
 		load_module mdd/mdd
+		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
+			#
+			# This block shall be moved up beside osd-zfs as soon
+			# as osd-ldiskfs stops using mdd symbols.
+			#
+			grep -q exportfs_decode_fh $SYMLIST ||
+				{ modprobe exportfs 2> /dev/null || true; }
+			load_module ../ldiskfs/ldiskfs/ldiskfs
+			load_module lvfs/fsfilt_ldiskfs
+			load_module osd-ldiskfs/osd_ldiskfs
+		fi
 		load_module mdt/mdt
-		load_module lvfs/fsfilt_ldiskfs
 		load_module cmm/cmm
-		load_module osd-ldiskfs/osd_ldiskfs
 		load_module ost/ost
 		if [ "$USE_OFD" == yes ]; then
 			load_module ofd/ofd
@@ -753,7 +760,7 @@ get_obdfilter_param() {
 	local name=$3
 
 	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
-		osd-*.$device.$name" 2>&1 | grep -v 'Found no match'
+		osd-*.$device.$name 2>&1" | grep -v 'Found no match'
 }
 
 set_obdfilter_param() {
@@ -763,7 +770,7 @@ set_obdfilter_param() {
 	local value=$4
 
 	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
-		osd-*.$device.$name=$value" 2>&1 | grep -v 'Found no match'
+		osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
 }
 
 set_debug_size () {
@@ -912,6 +919,39 @@ start() {
     return $RC
 }
 
+#
+# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed".
+# Because stopping corresponding target may not clear this in-memory state, we
+# need to zap the pool from memory by exporting and reimporting the pool.
+#
+# Although the uberblocks are not updated when a pool is freezed, transactions
+# are still written to the disks.  Modified blocks may be cached in memory when
+# tests try reading them back.  The export-and-reimport process also evicts any
+# cached pool data from memory to provide the correct "data loss" semantics.
+#
+refresh_disk() {
+	local facet=$1
+	local fstype=$(facet_fstype $facet)
+	local _dev
+	local dev
+	local poolname
+
+	if [ "${fstype}" == "zfs" ]; then
+		_dev=$(facet_active $facet)_dev
+		dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev}
+		poolname="${dev%%/*}" # poolname is string before "/"
+
+		if [ "${poolname}" == "" ]; then
+			echo "invalid dataset name: $dev"
+			return
+		fi
+		do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back"
+		do_facet $facet "$ZPOOL export ${poolname}"
+		do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \
+		                 ${poolname}"
+	fi
+}
+
 stop() {
     local running
     local facet=$1
@@ -1276,12 +1316,13 @@ remount_facet() {
 }
 
 reboot_facet() {
-    local facet=$1
-    if [ "$FAILURE_MODE" = HARD ]; then
-        reboot_node $(facet_active_host $facet)
-    else
-        sleep 10
-    fi
+	local facet=$1
+	if [ "$FAILURE_MODE" = HARD ]; then
+		reboot_node $(facet_active_host $facet)
+	else
+		refresh_disk ${facet}
+		sleep 10
+	fi
 }
 
 boot_node() {
@@ -1829,14 +1870,6 @@ facet_failover() {
 
     echo "Failing $facet on node $host"
 
-    # Make sure the client data is synced to disk. LU-924
-    #
-    # We don't write client data synchrnously (to avoid flooding sync writes
-    # when there are many clients connecting), so if the server reboots before
-    # the client data reachs disk, the client data will be lost and the client
-    # will be evicted after recovery, which is not what we expected.
-    do_facet $facet "sync; sync; sync"
-
     local affected=$(affected_facets $facet)
 
     shutdown_facet $facet
@@ -1921,13 +1954,14 @@ fail_nodf() {
 }
 
 fail_abort() {
-    local facet=$1
-    stop $facet
-    change_active $facet
-    wait_for_facet $facet
-    mount_facet $facet -o abort_recovery
-    clients_up || echo "first df failed: $?"
-    clients_up || error "post-failover df: $?"
+	local facet=$1
+	stop $facet
+	refresh_disk ${facet}
+	change_active $facet
+	wait_for_facet $facet
+	mount_facet $facet -o abort_recovery
+	clients_up || echo "first df failed: $?"
+	clients_up || error "post-failover df: $?"
 }
 
 do_lmc() {
@@ -2219,6 +2253,8 @@ single_local_node () {
 get_env_vars() {
 	local var
 	local value
+	local facets=$(get_facets)
+	local facet
 
 	for var in ${!MODOPTS_*}; do
 		value=${!var}
@@ -2226,6 +2262,23 @@ get_env_vars() {
 	done
 
 	echo -n " USE_OFD=$USE_OFD"
+
+	for facet in ${facets//,/ }; do
+		var=${facet}_FSTYPE
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE; do
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n " FSTYPE=$FSTYPE"
+	fi
 }
 
 do_nodes() {
@@ -2278,6 +2331,25 @@ do_facet() {
     do_node $HOST "$@"
 }
 
+# Function: do_facet_random_file $FACET $FILE $SIZE
+# Creates FILE with random content on the given FACET of given SIZE
+
+do_facet_random_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/urandom of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
+do_facet_create_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/zero of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
 do_nodesv() {
     do_nodes --verbose "$@"
 }
@@ -3513,6 +3585,17 @@ stop_full_debug_logging() {
     debugrestore
 }
 
+# prints bash call stack
+log_trace_dump() {
+	echo "  Trace dump:"
+	for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do
+		local s=${BASH_SOURCE[$i]}
+		local l=${BASH_LINENO[$i-1]}
+		local f=${FUNCNAME[$i]}
+		echo "  = $s:$l:$f()"
+	done
+}
+
 ##################################
 # Test interface
 ##################################
@@ -3528,6 +3611,7 @@ error_noexit() {
     fi
 
     log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ "
+    log_trace_dump
 
     mkdir -p $LOGDIR
     # We need to dump the logs on all nodes
@@ -3541,11 +3625,11 @@ error_noexit() {
 }
 
 exit_status () {
-    local status=0
-    local log=$TESTSUITELOG
+	local status=0
+	local log=$TESTSUITELOG
 
-    [ -f "$log" ] && grep -q FAIL: $log && status=1
-    exit $status
+	[ -f "$log" ] && grep -q FAIL $log && status=1
+	exit $status
 }
 
 error() {
@@ -3775,7 +3859,7 @@ banner() {
 run_one() {
     local testnum=$1
     local message=$2
-    tfile=f${testnum}
+    tfile=f.${TESTSUITE}.${testnum}
     export tdir=d0.${TESTSUITE}/d${base}
     export TESTNAME=test_$testnum
     local SAVE_UMASK=`umask`
@@ -3801,34 +3885,34 @@ run_one() {
 #  - test result is saved to data file
 #
 run_one_logged() {
-    local BEFORE=`date +%s`
-    local TEST_ERROR
-    local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
-    local test_log=$LOGDIR/$name
-    rm -rf $LOGDIR/err
-    local SAVE_UMASK=`umask`
-    umask 0022
-
-    echo
-    log_sub_test_begin test_${1}
-    (run_one $1 "$2") 2>&1 | tee -i $test_log
-    local RC=${PIPESTATUS[0]}
-
-    [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
-        echo "test_$1 returned $RC" | tee $LOGDIR/err
-
-    duration=$((`date +%s` - $BEFORE))
-    pass "$1" "(${duration}s)"
-    [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
-    log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
-
-    if [ -f $LOGDIR/err ]; then
-        $FAIL_ON_ERROR && exit $RC
-    fi
+	local BEFORE=`date +%s`
+	local TEST_ERROR
+	local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
+	local test_log=$LOGDIR/$name
+	rm -rf $LOGDIR/err
+	local SAVE_UMASK=`umask`
+	umask 0022
+
+	echo
+	log_sub_test_begin test_${1}
+	(run_one $1 "$2") 2>&1 | tee -i $test_log
+	local RC=${PIPESTATUS[0]}
+
+	[ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
+		echo "test_$1 returned $RC" | tee $LOGDIR/err
+
+	duration=$((`date +%s` - $BEFORE))
+	pass "$1" "(${duration}s)"
+	[ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
+	log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+
+	if [ -f $LOGDIR/err ]; then
+		$FAIL_ON_ERROR && exit $RC
+	fi
 
-    umask $SAVE_UMASK
+	umask $SAVE_UMASK
 
-    return $RC
+	return 0
 }
 
 #