LU-2863 tests: fix lfsck/OI_scrub test scripts issues

1) sanity-scrub.sh checks Lustre version after Lustre initialization.

2) Re-calculate the expactation for lfsck/OI_scrub speed test:
   2.1) 1.1 * the theoretical value for speed upper limit.
   2.2) 0.9 * the theoretical value for speed lower limit.

3) Reformat the device before running sanity-scrub test_11.

4) Other cleanup.

Test-Parameters: testlist=sanity-scrub,sanity-lfsck

Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: Ib811fa24ac7581c341d596afbed064a4fc3e7357
Reviewed-on: http://review.whamcloud.com/5551
Tested-by: Hudson
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh
index 5e454d6..4d39cae 100644 (file)
--- a/lustre/tests/sanity-lfsck.sh
+++ b/lustre/tests/sanity-lfsck.sh
@@ -480,7 +480,7 @@ test_6b() {
        local POSITION1=$($SHOW_NAMESPACE |
                          awk '/^latest_start_position/ { print $4 }')
        if [ $POSITION0 -gt $POSITION1 ]; then
        local POSITION1=$($SHOW_NAMESPACE |
                          awk '/^latest_start_position/ { print $4 }')
        if [ $POSITION0 -gt $POSITION1 ]; then

-               [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] ||
+               [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||

                error "(7) Expect larger than: $POSITION0, but got $POSITION1"
        fi
 
                error "(7) Expect larger than: $POSITION0, but got $POSITION1"
        fi
 


@@ -726,9 +726,13 @@ test_9a() {
 
        local SPEED=$($SHOW_NAMESPACE |
                      awk '/^average_speed_phase1/ { print $2 }')
 
        local SPEED=$($SHOW_NAMESPACE |
                      awk '/^average_speed_phase1/ { print $2 }')

-       # (100 * (10 + 1)) / 10 = 110
-       [ $SPEED -lt 120 ] ||
-               error "(4) Unexpected speed $SPEED, should not more than 120"
+       # There may be time error, normally it should be less than 2.
+       # We allow another 10% schedule error.
+       #
+       # SPEED1 = (100 * (time + 2)) / time * 1.1
+       SPEED1=$((100 * (10 + 2) / 10 * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(4) Got speed $SPEED, expected less than $SPEED1"

 
        # adjust speed limit
        do_facet $SINGLEMDS \
 
        # adjust speed limit
        do_facet $SINGLEMDS \


@@ -736,13 +740,15 @@ test_9a() {
        sleep 10
 
        SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
        sleep 10
 
        SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')

-       # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180
-       [ $SPEED -lt 170 ] &&
-               error "(5) Unexpected speed $SPEED, should not less than 170"
+       # SPEED1=(100 * (time1 - 2) + 300 * (time2 - 2)) / (time1 + time2) * 0.9
+       SPEED1=$(((100 * (10 - 2) + 300 * (10 - 2)) / (10 + 10) * 9 / 10))
+       [ $SPEED -gt $SPEED1 ] ||
+               error "(5) Got speed $SPEED, expected more than $SPEED1"

 
 

-       # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220
-       [ $SPEED -lt 230 ] ||
-               error "(6) Unexpected speed $SPEED, should not more than 230"
+       # SPEED1=(100 * (time1 + 2) + 300 * (time2 + 2)) / (time1 + time2) * 1.1
+       SPEED1=$(((100 * (10 + 2) + 300 * (10 + 2)) / (10 + 10) * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(6) Got speed $SPEED, expected less than $SPEED1"

 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0


@@ -800,9 +806,13 @@ test_9b() {
 
        local SPEED=$($SHOW_NAMESPACE |
                      awk '/^average_speed_phase2/ { print $2 }')
 
        local SPEED=$($SHOW_NAMESPACE |
                      awk '/^average_speed_phase2/ { print $2 }')

-       # (50 * (10 + 1)) / 10 = 55
-       [ $SPEED -lt 60 ] ||
-               error "(8) Unexpected speed $SPEED, should not more than 60"
+       # There may be time error, normally it should be less than 2.
+       # We allow another 10% schedule error.
+       #
+       # SPEED1 = (50 * (time + 2)) / time * 1.1
+       SPEED1=$((50 * (10 + 2) / 10 * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(8) Got speed $SPEED, expected less than $SPEED1"

 
        # adjust speed limit
        do_facet $SINGLEMDS \
 
        # adjust speed limit
        do_facet $SINGLEMDS \


@@ -810,13 +820,15 @@ test_9b() {
        sleep 10
 
        SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
        sleep 10
 
        SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')

-       # (50 * (10 - 1) + 150 * (10 - 1)) / 20 = 90
-       [ $SPEED -lt 85 ] &&
-               error "(9) Unexpected speed $SPEED, should not less than 85"
-
-       # (50 * (10 + 1) + 150 * (10 + 1)) / 20 = 110
-       [ $SPEED -lt 115 ] ||
-               error "(10) Unexpected speed $SPEED, should not more than 115"
+       # SPEED1=(50 * (time1 - 2) + 150 * (time2 - 2)) / (time1 + time2) * 0.9
+       SPEED1=$(((50 * (10 - 2) + 150 * (10 - 2)) / (10 + 10) * 9 / 10))
+       [ $SPEED -gt $SPEED1 ] ||
+               error "(9) Got speed $SPEED, expected more than $SPEED1"
+
+       # SPEED1=(50 * (time1 + 2) + 150 * (time2 + 2)) / (time1 + time2) * 1.1
+       SPEED1=$(((50 * (10 + 2) + 150 * (10 + 2)) / (10 + 10) * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(10) Got speed $SPEED, expected less than $SPEED1"

 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0

diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 76727d9..45bc281 100644 (file)
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -17,10 +17,6 @@ init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 init_logging
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 init_logging
 

-[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
-       skip "test OI scrub only for ldiskfs" && exit 0
-[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
-       skip "Need MDS version at least 2.2.90" && exit 0

 require_dsh_mds || exit 0
 
 SAVED_MDSSIZE=${MDSSIZE}
 require_dsh_mds || exit 0
 
 SAVED_MDSSIZE=${MDSSIZE}


@@ -31,6 +27,14 @@ MDSSIZE=100000
 OSTSIZE=100000
 
 check_and_setup_lustre
 OSTSIZE=100000
 
 check_and_setup_lustre

+
+[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
+       skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
+       exit 0
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
+       skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
+       exit 0
+

 build_test_filter
 
 MDT_DEV="${FSNAME}-MDT0000"
 build_test_filter
 
 MDT_DEV="${FSNAME}-MDT0000"


@@ -563,12 +567,15 @@ test_9() {
        [ "$STATUS" == "scanning" ] ||
                error "(9) Expect 'scanning', but got '$STATUS'"
 
        [ "$STATUS" == "scanning" ] ||
                error "(9) Expect 'scanning', but got '$STATUS'"
 

-       # Do NOT ignore that there are 1024 pre-fetched items.
-       # So the max speed may be (1024 + 100 * 10) / 10.
-       # And there may be time error, so the max speed may be more large.

        local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
        local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')

-       [ $SPEED -gt 220 ] &&
-               error "(10) Unexpected speed $SPEED, should not more than 220"
+       # Do NOT ignore that there are 1024 pre-fetched items.
+       # And there may be time error, normally it should be less than 2.
+       # We allow another 10% schedule error.
+       #
+       # SPEED1 = (pre-fetched + 100 * (time - 2)) / time * 1.1
+       local SPEED1=$(((1024 + 100 * (10 + 2)) / 10 * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(10) Got speed $SPEED, expected less than $SPEED1"

 
        # adjust speed limit
        do_facet $SINGLEMDS \
 
        # adjust speed limit
        do_facet $SINGLEMDS \


@@ -576,11 +583,19 @@ test_9() {
        sleep 10
 
        SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
        sleep 10
 
        SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')

-       [ $SPEED -lt 220 ] &&
-               error "(11) Unexpected speed $SPEED, should not less than 220"
-
-       [ $SPEED -gt 300 ] &&
-               error "(12) Unexpected speed $SPEED, should not more than 300"
+       # SPEED1 = (pre-fetched + 100 * (time1 - 2) + 300 * (time2 - 2)) / \
+       #          (time1 + time2) * 0.9
+       SPEED1=$(((1024 + 100 * (10 - 2) + 300 * (10 - 2)) / \
+                 (10 + 10) * 9 / 10))
+       [ $SPEED -gt $SPEED1 ] ||
+               error "(11) Got speed $SPEED, expected more than $SPEED1"
+
+       # SPEED1 = (pre-fetched + 100 * (time1 + 2) + 300 * (time2 + 2)) / \
+       #          (time1 + time2) * 1.1
+       SPEED1=$(((1024 + 100 * (10 + 2) + 300 * (10 + 2)) / \
+                 (10 + 10) * 11 / 10))
+       [ $SPEED -lt $SPEED1 ] ||
+               error "(12) Got speed $SPEED, expected less than $SPEED1"

 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
 
        do_facet $SINGLEMDS \
                $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0


@@ -720,6 +735,8 @@ run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
 test_11() {
        echo "stopall"
        stopall > /dev/null
 test_11() {
        echo "stopall"
        stopall > /dev/null

+       echo "formatall"
+       formatall > /dev/null

        echo "setupall"
        setupall > /dev/null
 
        echo "setupall"
        setupall > /dev/null
 


@@ -741,8 +758,8 @@ test_11() {
        local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
        # notice we're creating a new llog for every OST on every startup
        # new features can make this even less stable, so we only check
        local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
        # notice we're creating a new llog for every OST on every startup
        # new features can make this even less stable, so we only check

-       # that the number of skipped files is less than 1.5x the number of files
-       local MAXIMUM=$((CREATED * 3 / 2))
+       # that the number of skipped files is less than 2x the number of files
+       local MAXIMUM=$((CREATED * 2))

        local MINIMUM=$((CREATED + 1)) # files + directory
        [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
        error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
        local MINIMUM=$((CREATED + 1)) # files + directory
        [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
        error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"