X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-scrub.sh;h=76727d91be3cac58ea6cb3038fc68a9d12a30d43;hb=3e1cff73c9abe513cb691ff0e81da332a6434a94;hp=4d1c1144cb1b6c8bfdbd876d16bc6a52db33ee01;hpb=d01d4c697a3c4423587159d58da6e455a5a3551f;p=fs%2Flustre-release.git

diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 4d1c114..76727d9 100644
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -17,8 +17,10 @@ init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 init_logging
 
-[ "${MDSFSTYPE:-$FSTYPE}" != "ldiskfs" ] &&
+[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
 	skip "test OI scrub only for ldiskfs" && exit 0
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
+	skip "Need MDS version at least 2.2.90" && exit 0
 require_dsh_mds || exit 0
 
 SAVED_MDSSIZE=${MDSSIZE}
@@ -33,8 +35,12 @@ build_test_filter
 
 MDT_DEV="${FSNAME}-MDT0000"
 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
+START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
+STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
 SHOW_SCRUB="do_facet $SINGLEMDS \
 		$LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
+MOUNT_OPTS_SCRUB="-o user_xattr"
+MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
 
 scrub_prep() {
 	local nfiles=$1
@@ -59,14 +65,14 @@ scrub_prep() {
 test_0() {
 	scrub_prep 0
 	echo "start $SINGLEMDS without disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
 		error "(1) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(2) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
 
 	mount_client $MOUNT || error "(4) Fail to start client!"
@@ -78,14 +84,54 @@ run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
 
 test_1a() {
 	scrub_prep 0
+	echo "start $SINGLEMDS without disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+		error "(1) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(2) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
+
+	mount_client $MOUNT || error "(4) Fail to start client!"
+
+	#define OBD_FAIL_OSD_FID_MAPPING			0x193
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
+	# update .lustre OI mapping
+	touch $MOUNT/.lustre
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+	umount_client $MOUNT || error "(5) Fail to stop client!"
+
+	echo "stop $SINGLEMDS"
+	stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(7) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(8) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ "$FLAGS" == "inconsistent" ] ||
+		error "(9) Expect 'inconsistent', but got '$FLAGS'"
+}
+run_test 1a "Auto trigger initial OI scrub when server mounts"
+
+test_1b() {
+	scrub_prep 0
 	mds_remove_ois || error "(1) Fail to remove/recreate!"
 
 	echo "start $SINGLEMDS without disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
 	sleep 3
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(3) Expect 'completed', but got '$STATUS'"
 
@@ -94,18 +140,53 @@ test_1a() {
 	diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
 		error "(5) File diff failed unexpected!"
 }
-run_test 1a "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
+run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
+
+test_1c() {
+	local index
+
+	# OI files to be removed:
+	# idx 0: oi.16.0
+	# idx 1: oi.16.1
+	# idx 2: oi.16.{2,4,8,16,32}
+	# idx 3: oi.16.{3,9,27}
+	# idx 5: oi.16.{5,25}
+	# idx 7: oi.16.{7,49}
+	for index in 0 1 2 3 5 7; do
+		scrub_prep 0
+		mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
+
+		echo "start $SINGLEMDS with disabling OI scrub"
+		start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
+			/dev/null || error "(2) Fail to start MDS!"
+
+		local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+		[ "$FLAGS" == "recreated" ] ||
+			error "(3) Expect 'recreated', but got '$FLAGS'"
+
+		$START_SCRUB || error "(4) Fail to start OI scrub!"
+		sleep 3
+		local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+		[ "$STATUS" == "completed" ] ||
+			error "(5) Expect 'completed', but got '$STATUS'"
+
+		FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+		[ -z "$FLAGS" ] ||
+			error "(6) Expect empty flags, but got '$FLAGS'"
+	done
+}
+run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
 
 test_2() {
 	scrub_prep 0
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS without disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
 	sleep 3
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(3) Expect 'completed', but got '$STATUS'"
 
@@ -121,14 +202,14 @@ test_3() {
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(3) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent" ] ||
 		error "(4) Expect 'inconsistent', but got '$FLAGS'"
 	echo "stopall"
@@ -141,14 +222,14 @@ test_4() {
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(3) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent" ] ||
 		error "(4) Expect 'inconsistent', but got '$FLAGS'"
 
@@ -160,7 +241,7 @@ test_4() {
 		error "(6) File diff failed unexpected!"
 
 	sleep 3
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(7) Expect 'completed', but got '$STATUS'"
 }
@@ -171,14 +252,14 @@ test_5() {
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(3) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent" ] ||
 		error "(4) Expect 'inconsistent', but got '$FLAGS'"
 
@@ -186,7 +267,7 @@ test_5() {
 
 	do_facet $SINGLEMDS \
 		$LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
@@ -194,65 +275,65 @@ test_5() {
 
 	umount_client $MOUNT || error "(7) Fail to stop client!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(8) Expect 'scanning', but got '$STATUS'"
 
-#define OBD_FAIL_OSD_SCRUB_CRASH	 0x191
+	#define OBD_FAIL_OSD_SCRUB_CRASH	 0x191
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
 	sleep 4
 	echo "stop $SINGLEMDS"
 	stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(10) Fail to start MDS!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "crashed" ] ||
 		error "(11) Expect 'crashed', but got '$STATUS'"
 
 	echo "stop $SINGLEMDS"
 	stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
 
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	echo "start $SINGLEMDS without disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
 		error "(13) Fail to start MDS!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(14) Expect 'scanning', but got '$STATUS'"
 
-#define OBD_FAIL_OSD_SCRUB_FATAL	 0x192
+	#define OBD_FAIL_OSD_SCRUB_FATAL	 0x192
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
 	sleep 4
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "failed" ] ||
 		error "(15) Expect 'failed', but got '$STATUS'"
 
 	mount_client $MOUNT || error "(16) Fail to start client!"
 
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	stat $DIR/$tdir/${tfile}1000 ||
 		error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(18) Expect 'scanning', but got '$STATUS'"
 
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
 	do_facet $SINGLEMDS $LCTL set_param fail_val=0
 	sleep 5
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(19) Expect 'completed', but got '$STATUS'"
 
-	FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
 }
 run_test 5 "OI scrub state machine"
@@ -262,14 +343,14 @@ test_6() {
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(3) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent" ] ||
 		error "(4) Expect 'inconsistent', but got '$FLAGS'"
 
@@ -277,64 +358,69 @@ test_6() {
 
 	do_facet $SINGLEMDS \
 		$LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
 		error "(6) File diff failed unexpected!"
 
-	# Fail the OI scrub to guarantee there is at least on checkpoint
-#define OBD_FAIL_OSD_SCRUB_FATAL	 0x192
+	# Sleep 5 sec to guarantee at least one object processed by OI scrub
+	sleep 5
+	# Fail the OI scrub to guarantee there is at least one checkpoint
+	#define OBD_FAIL_OSD_SCRUB_FATAL	 0x192
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
 	sleep 4
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "failed" ] ||
 		error "(7) Expect 'failed', but got '$STATUS'"
 
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
+	# stat will re-trigger OI scrub
 	stat $DIR/$tdir/${tfile}800 ||
 		error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
 
 	umount_client $MOUNT || error "(9) Fail to stop client!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(10) Expect 'scanning', but got '$STATUS'"
 
-#define OBD_FAIL_OSD_SCRUB_CRASH	 0x191
+	#define OBD_FAIL_OSD_SCRUB_CRASH	 0x191
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
 	sleep 4
-	local POSITION0=$($SHOW_SCRUB | sed -n '11'p | awk '{print $2}')
+	local POSITION0=$($SHOW_SCRUB |
+			awk '/^last_checkpoint_position/ {print $2}')
 	POSITION0=$((POSITION0 + 1))
 
 	echo "stop $SINGLEMDS"
 	stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
 
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	echo "start $SINGLEMDS without disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
 		error "(12) Fail to start MDS!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(13) Expect 'scanning', but got '$STATUS'"
 
-	local POSITION1=$($SHOW_SCRUB | sed -n '10'p |awk '{print $2}')
+	local POSITION1=$($SHOW_SCRUB |
+			awk '/^latest_start_position/ {print $2}')
 	[ $POSITION0 -eq $POSITION1 ] ||
 		error "(14) Expect position: $POSITION0, but got $POSITION1"
 
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
 	do_facet $SINGLEMDS $LCTL set_param fail_val=0
 	sleep 5
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(15) Expect 'completed', but got '$STATUS'"
 
-	FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
 }
 run_test 6 "OI scrub resumes from last checkpoint"
@@ -344,14 +430,14 @@ test_7() {
 	mds_backup_restore || error "(1) Fail to backup/restore!"
 
 	echo "start $SINGLEMDS with disabling OI scrub"
-	start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
 		error "(2) Fail to start MDS!"
 
-	local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "init" ] ||
 		error "(3) Expect 'init', but got '$STATUS'"
 
-	local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent" ] ||
 		error "(4) Expect 'inconsistent', but got '$FLAGS'"
 
@@ -359,7 +445,7 @@ test_7() {
 
 	do_facet $SINGLEMDS \
 		$LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
-#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
 	do_facet $SINGLEMDS $LCTL set_param fail_val=3
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
 	diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
@@ -368,33 +454,314 @@ test_7() {
 	stat $DIR/$tdir/${tfile}300 ||
 		error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
 
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "scanning" ] ||
 		error "(8) Expect 'scanning', but got '$STATUS'"
 
-	FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ "$FLAGS" == "inconsistent,auto" ] ||
 		error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
 
 	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
 	do_facet $SINGLEMDS $LCTL set_param fail_val=0
 	sleep 5
-	STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
 	[ "$STATUS" == "completed" ] ||
 		error "(10) Expect 'completed', but got '$STATUS'"
 
-	FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
 	[ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
 }
 run_test 7 "System is available during OI scrub scanning"
 
-# restore the ${facet}_MKFS_OPTS variables
-for facet in MGS MDS OST; do
-	opts=SAVED_${facet}_MKFS_OPTS
-	if [[ -n ${!opts} ]]; then
-		eval ${facet}_MKFS_OPTS=\"${!opts}\"
+test_8() {
+	scrub_prep 0
+	mds_backup_restore || error "(1) Fail to backup/restore!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(2) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(3) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ "$FLAGS" == "inconsistent" ] ||
+		error "(4) Expect 'inconsistent', but got '$FLAGS'"
+
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	do_facet $SINGLEMDS $LCTL set_param fail_val=3
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
+	$START_SCRUB || error "(5) Fail to start OI scrub!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(6) Expect 'scanning', but got '$STATUS'"
+
+	$STOP_SCRUB || error "(7) Fail to stop OI scrub!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "stopped" ] ||
+		error "(8) Expect 'stopped', but got '$STATUS'"
+
+	$START_SCRUB || error "(9) Fail to start OI scrub!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(10) Expect 'scanning', but got '$STATUS'"
+
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+	do_facet $SINGLEMDS $LCTL set_param fail_val=0
+	sleep 5
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(11) Expect 'completed', but got '$STATUS'"
+
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
+}
+run_test 8 "Control OI scrub manually"
+
+test_9() {
+	if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
+		skip "Testing on UP system, the speed may be inaccurate."
+		return 0
 	fi
-done
+
+	scrub_prep 8000
+	mds_backup_restore || error "(1) Fail to backup/restore!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(2) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(3) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ "$FLAGS" == "inconsistent" ] ||
+		error "(4) Expect 'inconsistent', but got '$FLAGS'"
+
+	# OI scrub should run with full speed under inconsistent case
+	$START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
+
+	sleep 10
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(6) Expect 'completed', but got '$STATUS'"
+
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
+
+	# OI scrub should run with limited speed under non-inconsistent case
+	$START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
+
+	sleep 10
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(9) Expect 'scanning', but got '$STATUS'"
+
+	# Do NOT ignore that there are 1024 pre-fetched items.
+	# So the max speed may be (1024 + 100 * 10) / 10.
+	# And there may be time error, so the max speed may be more large.
+	local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
+	[ $SPEED -gt 220 ] &&
+		error "(10) Unexpected speed $SPEED, should not more than 220"
+
+	# adjust speed limit
+	do_facet $SINGLEMDS \
+		$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
+	sleep 10
+
+	SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
+	[ $SPEED -lt 220 ] &&
+		error "(11) Unexpected speed $SPEED, should not less than 220"
+
+	[ $SPEED -gt 300 ] &&
+		error "(12) Unexpected speed $SPEED, should not more than 300"
+
+	do_facet $SINGLEMDS \
+		$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+	sleep 6
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(13) Expect 'completed', but got '$STATUS'"
+}
+run_test 9 "OI scrub speed control"
+
+test_10a() {
+	scrub_prep 0
+	mds_backup_restore || error "(1) Fail to backup/restore!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(2) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(3) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ "$FLAGS" == "inconsistent" ] ||
+		error "(4) Expect 'inconsistent', but got '$FLAGS'"
+
+	mount_client $MOUNT || error "(5) Fail to start client!"
+
+	do_facet $SINGLEMDS \
+		$LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	do_facet $SINGLEMDS $LCTL set_param fail_val=3
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
+	diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
+		error "(6) File diff failed unexpected!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(7) Expect 'scanning', but got '$STATUS'"
+
+	umount_client $MOUNT || error "(8) Fail to stop client!"
+
+	echo "stop $SINGLEMDS"
+	stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(10) Fail to start MDS!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "paused" ] ||
+		error "(11) Expect 'paused', but got '$STATUS'"
+
+	echo "stop $SINGLEMDS"
+	stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
+
+	echo "start $SINGLEMDS without disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+		error "(13) Fail to start MDS!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(14) Expect 'scanning', but got '$STATUS'"
+
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+	do_facet $SINGLEMDS $LCTL set_param fail_val=0
+	sleep 5
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(15) Expect 'completed', but got '$STATUS'"
+
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
+}
+run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
+
+test_10b() {
+	scrub_prep 0
+	mds_backup_restore || error "(1) Fail to backup/restore!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(2) Fail to start MDS!"
+
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "init" ] ||
+		error "(3) Expect 'init', but got '$STATUS'"
+
+	local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ "$FLAGS" == "inconsistent" ] ||
+		error "(4) Expect 'inconsistent', but got '$FLAGS'"
+
+	#define OBD_FAIL_OSD_SCRUB_DELAY	 0x190
+	do_facet $SINGLEMDS $LCTL set_param fail_val=3
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
+
+	$START_SCRUB || error "(5) Fail to start OI scrub!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(6) Expect 'scanning', but got '$STATUS'"
+
+	echo "stop $SINGLEMDS"
+	stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
+
+	echo "start $SINGLEMDS with disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+		error "(8) Fail to start MDS!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "paused" ] ||
+		error "(9) Expect 'paused', but got '$STATUS'"
+
+	echo "stop $SINGLEMDS"
+	stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
+
+	echo "start $SINGLEMDS without disabling OI scrub"
+	start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+		error "(11) Fail to start MDS!"
+
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "scanning" ] ||
+		error "(12) Expect 'scanning', but got '$STATUS'"
+
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+	do_facet $SINGLEMDS $LCTL set_param fail_val=0
+	sleep 5
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(13) Expect 'completed', but got '$STATUS'"
+
+	FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
+	[ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
+}
+run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
+
+test_11() {
+	echo "stopall"
+	stopall > /dev/null
+	echo "setupall"
+	setupall > /dev/null
+
+	local CREATED=100
+	local tname=`date +%s`
+	rm -rf $MOUNT/$tname > /dev/null
+	mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
+
+	createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
+
+	# reset OI scrub start point by force
+	$START_SCRUB -r || error "(3) Fail to start OI scrub!"
+	sleep 3
+	local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(4) Expect 'completed', but got '$STATUS'"
+
+	# OI scrub should skip the new created objects for the first accessing
+	local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
+	# notice we're creating a new llog for every OST on every startup
+	# new features can make this even less stable, so we only check
+	# that the number of skipped files is less than 1.5x the number of files
+	local MAXIMUM=$((CREATED * 3 / 2))
+	local MINIMUM=$((CREATED + 1)) # files + directory
+	[ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
+	error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
+
+	# reset OI scrub start point by force
+	$START_SCRUB -r || error "(6) Fail to start OI scrub!"
+	sleep 3
+	STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+	[ "$STATUS" == "completed" ] ||
+		error "(7) Expect 'completed', but got '$STATUS'"
+
+	# OI scrub should skip the new created object only once
+	SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
+	[ $SKIPPED -eq 0 ] ||
+		error "(8) Expect 0 objects skipped, but got $SKIPPED"
+
+	rm -rf $MOUNT/$tname > /dev/null
+}
+run_test 11 "OI scrub skips the new created objects only once"
 
 # restore MDS/OST size
 MDSSIZE=${SAVED_MDSSIZE}
@@ -403,5 +770,5 @@ OSTSIZE=${SAVED_OSTSIZE}
 # cleanup the system at last
 formatall
 
-complete $(basename $0) $SECONDS
+complete $SECONDS
 exit_status