LU-5075 test: keep LFSCK fail_loc until recovery completed

[fs/lustre-release.git] / lustre / tests / sanity-lfsck.sh

diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh
index 566bb53..ea3df21 100644 (file)
--- a/lustre/tests/sanity-lfsck.sh
+++ b/lustre/tests/sanity-lfsck.sh
@@ -19,7 +19,6 @@ init_logging
 
 require_dsh_mds || exit 0
 
-MCREATE=${MCREATE:-mcreate}
 SAVED_MDSSIZE=${MDSSIZE}
 SAVED_OSTSIZE=${OSTSIZE}
 SAVED_OSTCOUNT=${OSTCOUNT}
@@ -42,11 +41,7 @@ setupall
        ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
 
 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
-       ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
-
-[ $(facet_fstype $SINGLEMDS) = "zfs" ] &&
-# bug number for skipped test:        LU-4970
-       ALWAYS_EXCEPT="$ALWAYS_EXCEPT 14"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19 20 21"
 
 build_test_filter
 
@@ -514,7 +509,7 @@ test_6a() {
        local POS1=$($SHOW_NAMESPACE |
                     awk '/^latest_start_position/ { print $2 }' |
                     tr -d ',')
-       [ $POS0 -lt $POS1 ] ||
+       [[ $POS0 -lt $POS1 ]] ||
                error "(7) Expect larger than: $POS0, but got $POS1"
 
        do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
@@ -572,10 +567,10 @@ test_6b() {
                       awk '/^latest_start_position/ { print $4 }')
 
        if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then
-               [ $O_POS0 -lt $O_POS1 ] ||
+               [[ $O_POS0 -lt $O_POS1 ]] ||
                        error "(7.1) $O_POS1 is not larger than $O_POS0"
        else
-               [ $D_POS0 -lt $D_POS1 ] ||
+               [[ $D_POS0 -lt $D_POS1 ]] ||
                        error "(7.2) $D_POS1 is not larger than $D_POS0"
        fi
 
@@ -734,6 +729,21 @@ test_8()
        start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
                error "(14) Fail to start MDS!"
 
+       local timeout=$(max_recovery_time)
+       local timer=0
+
+       while [ $timer -lt $timeout ]; do
+               STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+                       mdt.${MDT_DEV}.recovery_status |
+                       awk '/^status/ { print \\\$2 }'")
+               [ "$STATUS" != "RECOVERING" ] && break;
+               sleep 1
+               timer=$((timer + 1))
+       done
+
+       [ $timer != $timeout ] ||
+               error "(14.1) recovery timeout"
+
        STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
        [ "$STATUS" == "crashed" ] ||
                error "(15) Expect 'crashed', but got '$STATUS'"
@@ -756,6 +766,19 @@ test_8()
        start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
                error "(19) Fail to start MDS!"
 
+       timer=0
+       while [ $timer -lt $timeout ]; do
+               STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+                       mdt.${MDT_DEV}.recovery_status |
+                       awk '/^status/ { print \\\$2 }'")
+               [ "$STATUS" != "RECOVERING" ] && break;
+               sleep 1
+               timer=$((timer + 1))
+       done
+
+       [ $timer != $timeout ] ||
+               error "(19.1) recovery timeout"
+
        STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
        [ "$STATUS" == "paused" ] ||
                error "(20) Expect 'paused', but got '$STATUS'"
@@ -1273,6 +1296,8 @@ test_14() {
        createmany -o $DIR/$tdir/f $((count + 32))
        do_facet ost1 $LCTL set_param fail_loc=0
 
+       start_full_debug_logging
+
        # exhaust other pre-created dangling cases
        count=$(precreated_ost_obj_count 0 0)
        createmany -o $DIR/$tdir/a $count ||
@@ -1316,6 +1341,7 @@ test_14() {
 
        echo "'ls' should success after layout LFSCK repairing"
        ls -ail $DIR/$tdir > /dev/null || error "(9) ls should success."
+       stop_full_debug_logging
 }
 run_test 14 "LFSCK can repair MDT-object with dangling reference"
 
@@ -1737,8 +1763,7 @@ test_18c() {
 
        if [ $MDSCOUNT -ge 2 ]; then
                $LFS mkdir -i 1 $DIR/$tdir/a2
-               $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
-               do_facet ost2 $LCTL set_param fail_loc=0x1617
+               $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a2
                dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
                $LFS getstripe $DIR/$tdir/a2/f2
        fi
@@ -1788,7 +1813,7 @@ test_18c() {
        done
 
        if [ $MDSCOUNT -ge 2 ]; then
-               expected=3
+               expected=2
        else
                expected=1
        fi
@@ -1807,7 +1832,7 @@ test_18c() {
                        error "(5) Expect 0 fixed on mds2, but got: $repaired"
        fi
 
-       echo "There should be some stub under .lustre/lost+found/MDT0001/"
+       echo "There should NOT be some stub under .lustre/lost+found/MDT0001/"
        ls -ail $MOUNT/.lustre/lost+found/MDT0001/*-N-0 &&
                error "(6) .lustre/lost+found/MDT0001/ should be empty"
 
@@ -2404,6 +2429,33 @@ test_20() {
 }
 run_test 20 "Handle the orphan with dummy LOV EA slot properly"
 
+test_21() {
+       [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.59) ]] &&
+               skip "ignore the test if MDS is older than 2.5.59" && exit 0
+
+       check_mount_and_prep
+       createmany -o $DIR/$tdir/f 100 || error "(0) Fail to create 100 files"
+
+       echo "Start all LFSCK components by default (-s 1)"
+       do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -s 1 -r ||
+               error "Fail to start LFSCK"
+
+       echo "namespace LFSCK should be in 'scanning-phase1' status"
+       local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+       [ "$STATUS" == "scanning-phase1" ] ||
+               error "Expect namespace 'scanning-phase1', but got '$STATUS'"
+
+       echo "layout LFSCK should be in 'scanning-phase1' status"
+       STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
+       [ "$STATUS" == "scanning-phase1" ] ||
+               error "Expect layout 'scanning-phase1', but got '$STATUS'"
+
+       echo "Stop all LFSCK components by default"
+       do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 ||
+               error "Fail to stop LFSCK"
+}
+run_test 21 "run all LFSCK components by default"
+
 $LCTL set_param debug=-lfsck > /dev/null || true
 
 # restore MDS/OST size