X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Frecovery-small.sh;h=c5166e917c234a2af1b2e00ac2f9278289afad3c;hb=49acd1450fff0f710924b008ab85fdb3f5f68015;hp=91d0b0fd877af73cfa44a9aafea7ab0d4d0f6d94;hpb=688d5da6a89882fdf9ebfb3e52ca869e9e959655;p=fs%2Flustre-release.git

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 91d0b0f..c5166e9 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -9,11 +9,6 @@ init_test_env $@
 init_logging
 
 ALWAYS_EXCEPT="$RECOVERY_SMALL_EXCEPT "
-if $SHARED_KEY; then
-	# bug number for skipped test: LU-12896
-	ALWAYS_EXCEPT+="               110k"
-	# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
-fi
 
 build_test_filter
 
@@ -1074,6 +1069,8 @@ test_26a() {      # was test_26 bug 5921 - evict dead exports by pinger
 
 	check_timeout || return 1
 
+	# make sure all imports are connected and not IDLE
+	do_facet client lfs df > /dev/null
 # OBD_FAIL_PTLRPC_DROP_RPC 0x505
 	do_facet client lctl set_param fail_loc=0x505
 	local before=$(date +%s)
@@ -1084,7 +1081,7 @@ test_26a() {      # was test_26 bug 5921 - evict dead exports by pinger
 	# the loser might have to wait for the next ping.
 	sleep $((TIMEOUT * 2 + TIMEOUT * 3 / 4))
 	do_facet client lctl set_param fail_loc=0x0
-	do_facet client df > /dev/null
+	do_facet client lfs df > /dev/null
 
 	local oscs=$(lctl dl | awk '/-osc-/ {print $4}')
 	check_clients_evicted $before ${oscs[@]}
@@ -1122,10 +1119,9 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
 	# = 9 * PING_INTERVAL + PING_INTERVAL
 	# = 10 PING_INTERVAL = 10 obd_timeout / 4 = 2.5 obd_timeout
 	# let's wait $((TIMEOUT * 3)) # bug 19887
-	local rc=0
-	wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 3)) || \
-		error "Client was not evicted by ost" rc=1
-	wait_client_evicted $SINGLEMDS $MDS_NEXP $((TIMEOUT * 3)) || \
+	wait_client_evicted ost1 $OST_NEXP $((TIMEOUT * 3)) ||
+		error "Client was not evicted by ost"
+	wait_client_evicted $SINGLEMDS $MDS_NEXP $((TIMEOUT * 3)) ||
 		error "Client was not evicted by mds"
 }
 run_test 26b "evict dead exports"
@@ -2894,7 +2890,7 @@ test_138() {
 	sleep 55
 	stop $SINGLEMDS || error "stop MDS failed"
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
-	start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) ||
+	start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) $MDS_MOUNT_OPTS ||
 		error "start MDS failed"
 	zconf_mount_clients $CLIENTS $MOUNT
 }
@@ -2933,7 +2929,7 @@ test_140a() {
 	mount_mds_client
 
 	local cnt
-	cnt=$(do_facet mds1 $LCTL get_param "mdt.*.exports.*.export" |
+	cnt=$(do_facet mds1 $LCTL get_param "mdt.*MDT0000.exports.*.export" |
 		grep export_flags.*no_recovery | wc -l)
 	echo "$cnt clients with recovery disabled"
 	umount_mds_client
@@ -2944,7 +2940,7 @@ test_140a() {
 	do_facet mds1 $LCTL set_param mdt.*.local_recovery=1
 	mount_mds_client
 
-	cnt=$(do_facet mds1 $LCTL get_param "mdt.*.exports.*.export" |
+	cnt=$(do_facet mds1 $LCTL get_param "mdt.*MDT0000.exports.*.export" |
 		grep export_flags.*no_recovery | wc -l)
 	echo "$cnt clients with recovery disabled"
 	umount_mds_client
@@ -2970,11 +2966,13 @@ test_140b() {
 	mount_mds_client
 	replay_barrier mds1
 	umount_mds_client
-	local before=$SECONDS
 	fail mds1
-	local after=$SECONDS
-	(( $after-$before < $TIMEOUT*2 )) ||
-		error "recovery took too long" $((after-bsfore)) $TIMEOUT
+	# Lustre: tfs-MDT0000: Recovery over after 0:03, of 2 clients 2 rec...
+	local recovery=$(do_facet mds1 dmesg |
+			 awk '/Recovery over after/ { print $6 }' | tail -1 |
+			 awk -F: '{ print $1 * 60 + $2 }')
+	(( recovery < TIMEOUT * 2 + 5 )) ||
+		error "recovery took too long $recovery > $((TIMEOUT * 2 + 5))"
 }
 run_test 140b "local mount is excluded from recovery"
 
@@ -3035,6 +3033,127 @@ test_143() {
 }
 run_test 143 "orphan cleanup thread shouldn't be blocked even delete failed"
 
+test_145() {
+	[ $MDSCOUNT -lt 3 ] && skip "needs >= 3 MDTs"
+	[ $(facet_active_host mds2) = $(facet_active_host mds3) ] &&
+		skip "needs mds2 and mds3 on separate nodes"
+
+	replay_barrier mds1
+
+	touch $DIR/$tfile
+
+#define OBD_FAIL_PTLRPC_DELAY_RECOV      0x507
+	echo block mds_connect from mds2
+	do_facet mds2 "$LCTL set_param fail_loc=0x507"
+
+#define OBD_FAIL_OUT_UPDATE_DROP	0x1707
+	echo block recovery updates from mds3
+	do_facet mds3 "$LCTL set_param fail_loc=0x1707"
+
+	local hard_timeout=\
+$(do_facet mds1 $LCTL get_param -n mdt.$FSNAME-MDT0000.recovery_time_hard)
+
+	fail mds1 &
+
+	local get_soft_timeout_cmd=\
+"$LCTL get_param -n mdt.$FSNAME-MDT0000.recovery_time_soft 2>/dev/null"
+
+	echo wait until mds1 recovery_time_soft is $hard_timeout
+	wait_update $(facet_host mds1) "$get_soft_timeout_cmd" \
+"$hard_timeout" $hard_timeout
+
+	echo unblock mds_connect from mds2
+	do_facet mds2 "$LCTL set_param fail_loc=0"
+
+	echo upblock recovery updates from mds3
+	do_facet mds3 "$LCTL set_param fail_loc=0"
+
+	wait
+	[ -f $DIR/$tfile ] || error "$DIR/$tfile does not exist"
+}
+run_test 145 "connect mdtlovs and process update logs after recovery expire"
+
+test_147() {
+	local obd_timeout=200
+	local old=$($LCTL get_param -n timeout)
+	local f=$DIR/$tfile
+	local connection_count
+
+	$LFS setstripe -i 0 -c 1 $f
+	stripe_index=$($LFS getstripe -i $f)
+	if [ $stripe_index -ne 0 ]; then
+	       $LFS getstripe $f
+	       error "$f: stripe_index $stripe_index != 0" && return
+	fi
+
+	$LCTL set_param timeout=$obd_timeout
+	stack_trap "$LCTL set_param timeout=$old && client_reconnect" EXIT
+
+	# OBD_FAIL_OST_CONNECT_NET2
+	# lost reply to connect request
+	do_facet ost1 lctl set_param fail_loc=0x00000225 timeout=$obd_timeout
+	stack_trap "do_facet ost1 $LCTL set_param fail_loc=0 timeout=$old" EXIT
+
+
+	ost_evict_client
+	# force reconnect
+	$LFS df $MOUNT > /dev/null 2>&1 &
+	sleep $((obd_timeout * 3 / 4))
+
+	$LCTL get_param osc.$FSNAME-OST0000-osc-*.state
+	connection_count=$($LCTL get_param osc.$FSNAME-OST0000-osc-*.state |
+			   tac |  sed "/FULL/,$ d" | grep CONNECTING | wc -l)
+
+	echo $connection_count
+	(($connection_count >= 6)) || error "Client reconnected too slow"
+}
+run_test 147 "Check client reconnect"
+
+test_148() {
+	local wce_param="obdfilter.$FSNAME-OST0000.writethrough_cache_enable"
+	local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+	local amc=$(at_max_get client)
+	local amo=$(at_max_get ost1)
+	local timeout
+
+	at_max_set 0 client
+	at_max_set 0 ost1
+	timeout=$(request_timeout client)
+
+	[ "$(facet_fstype ost1)" = "ldiskfs" ] && {
+		# save old r/o cache settings
+		save_lustre_params ost1 $wce_param > $p
+
+		# disable r/o cache
+		do_facet ost1 "$LCTL set_param -n $wce_param=0"
+	}
+
+	$LFS setstripe -i 0 -c 1 $DIR/$tfile
+	dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1 oflag=direct
+	cp $DIR/$tfile $TMP/$tfile
+	#define OBD_FAIL_OST_BRW_PAUSE_BULK2     0x227
+	do_facet ost1 $LCTL set_param fail_loc=0x80000227
+	do_facet ost1 $LCTL set_param fail_val=$((timeout+2))
+	dd if=/dev/urandom of=$DIR/$tfile bs=4096 count=1 conv=notrunc,fdatasync
+	dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1 conv=notrunc,fdatasync
+	sleep 2
+	cancel_lru_locks osc
+	cmp -b $DIR/$tfile $TMP/$tfile || error "wrong data"
+
+	rm -f $DIR/$tfile $TMP/$tfile
+
+	at_max_set $amc client
+	at_max_set $amo ost1
+
+	[ "$(facet_fstype ost1)" = "ldiskfs" ] && {
+		# restore initial r/o cache settings
+		restore_lustre_params < $p
+	}
+
+	return 0
+}
+run_test 148 "data corruption through resend"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status