X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-lfsck.sh;h=ea3df2155775b6bd5b4fa9da06295fed7e729b2a;hp=1c3b3df28de38b9032a4f0d94b6f2de08288f11c;hb=7bd946c100cda3fd86909647fdf8aace21ef2b29;hpb=46e82c4644108eac92ac7c813c16bd7dd3283a56 diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index 1c3b3df..ea3df21 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -19,7 +19,6 @@ init_logging require_dsh_mds || exit 0 -MCREATE=${MCREATE:-mcreate} SAVED_MDSSIZE=${MDSSIZE} SAVED_OSTSIZE=${OSTSIZE} SAVED_OSTCOUNT=${OSTCOUNT} @@ -42,7 +41,7 @@ setupall ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c" [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] && - ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19" + ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19 20 21" build_test_filter @@ -510,7 +509,7 @@ test_6a() { local POS1=$($SHOW_NAMESPACE | awk '/^latest_start_position/ { print $2 }' | tr -d ',') - [ $POS0 -lt $POS1 ] || + [[ $POS0 -lt $POS1 ]] || error "(7) Expect larger than: $POS0, but got $POS1" do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0 @@ -568,10 +567,10 @@ test_6b() { awk '/^latest_start_position/ { print $4 }') if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then - [ $O_POS0 -lt $O_POS1 ] || + [[ $O_POS0 -lt $O_POS1 ]] || error "(7.1) $O_POS1 is not larger than $O_POS0" else - [ $D_POS0 -lt $D_POS1 ] || + [[ $D_POS0 -lt $D_POS1 ]] || error "(7.2) $D_POS1 is not larger than $D_POS0" fi @@ -730,6 +729,21 @@ test_8() start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || error "(14) Fail to start MDS!" + local timeout=$(max_recovery_time) + local timer=0 + + while [ $timer -lt $timeout ]; do + STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \ + mdt.${MDT_DEV}.recovery_status | + awk '/^status/ { print \\\$2 }'") + [ "$STATUS" != "RECOVERING" ] && break; + sleep 1 + timer=$((timer + 1)) + done + + [ $timer != $timeout ] || + error "(14.1) recovery timeout" + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') [ "$STATUS" == "crashed" ] || error "(15) Expect 'crashed', but got '$STATUS'" @@ -752,6 +766,19 @@ test_8() start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null || error "(19) Fail to start MDS!" + timer=0 + while [ $timer -lt $timeout ]; do + STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \ + mdt.${MDT_DEV}.recovery_status | + awk '/^status/ { print \\\$2 }'") + [ "$STATUS" != "RECOVERING" ] && break; + sleep 1 + timer=$((timer + 1)) + done + + [ $timer != $timeout ] || + error "(19.1) recovery timeout" + STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') [ "$STATUS" == "paused" ] || error "(20) Expect 'paused', but got '$STATUS'" @@ -1269,6 +1296,8 @@ test_14() { createmany -o $DIR/$tdir/f $((count + 32)) do_facet ost1 $LCTL set_param fail_loc=0 + start_full_debug_logging + # exhaust other pre-created dangling cases count=$(precreated_ost_obj_count 0 0) createmany -o $DIR/$tdir/a $count || @@ -1312,6 +1341,7 @@ test_14() { echo "'ls' should success after layout LFSCK repairing" ls -ail $DIR/$tdir > /dev/null || error "(9) ls should success." + stop_full_debug_logging } run_test 14 "LFSCK can repair MDT-object with dangling reference" @@ -1684,11 +1714,11 @@ test_18b() { fi echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace" - mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 || - error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}" + mv $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0 $DIR/$tdir/a1/f1 || + error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0" if [ $MDSCOUNT -ge 2 ]; then - local name=$MOUNT/.lustre/lost+found/MDT0001/R-${fid2} + local name=$MOUNT/.lustre/lost+found/MDT0001/${fid2}-R-0 mv $name $DIR/$tdir/a2/f2 || error "(6) Fail to move $name" fi @@ -1733,8 +1763,7 @@ test_18c() { if [ $MDSCOUNT -ge 2 ]; then $LFS mkdir -i 1 $DIR/$tdir/a2 - $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2 - do_facet ost2 $LCTL set_param fail_loc=0x1617 + $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a2 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2 $LFS getstripe $DIR/$tdir/a2/f2 fi @@ -1784,7 +1813,7 @@ test_18c() { done if [ $MDSCOUNT -ge 2 ]; then - expected=3 + expected=2 else expected=1 fi @@ -1803,12 +1832,12 @@ test_18c() { error "(5) Expect 0 fixed on mds2, but got: $repaired" fi - echo "There should be some stub under .lustre/lost+found/MDT0001/" - ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* && + echo "There should NOT be some stub under .lustre/lost+found/MDT0001/" + ls -ail $MOUNT/.lustre/lost+found/MDT0001/*-N-0 && error "(6) .lustre/lost+found/MDT0001/ should be empty" echo "There should be some stub under .lustre/lost+found/MDT0000/" - ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* || + ls -ail $MOUNT/.lustre/lost+found/MDT0000/*-N-0 || error "(7) .lustre/lost+found/MDT0000/ should not be empty" } run_test 18c "Find out orphan OST-object and repair it (3)" @@ -1869,11 +1898,6 @@ test_18d() { awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || error "(3.0) MDS1 is not the expected 'scanning-phase2'" - # LU-3469: before osp_sync() is enabled, wait for a while to guarantee - # that former async repair operations have been executed on the OST(s). - sync - sleep 2 - do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0 for k in $(seq $MDSCOUNT); do @@ -2002,7 +2026,7 @@ test_18e() { error "(6) Expect 1 orphan has been fixed, but got: $repaired" echo "There should be stub file under .lustre/lost+found/MDT0000/" - local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*) + local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/*-C-0) [ ! -z $name ] || error "(7) .lustre/lost+found/MDT0000/ should not be empty" @@ -2077,6 +2101,361 @@ test_19b() { } run_test 19b "OST-object inconsistency self repair" +test_20() { + [ $OSTCOUNT -lt 2 ] && + skip "The test needs at least 2 OSTs" && return + + echo "#####" + echo "The target MDT-object and some of its OST-object are lost." + echo "The LFSCK should find out the left OST-objects and re-create" + echo "the MDT-object under the direcotry .lustre/lost+found/MDTxxxx/" + echo "with the partial OST-objects (LOV EA hole)." + + echo "New client can access the file with LOV EA hole via normal" + echo "system tools or commands without crash the system." + + echo "For old client, even though it cannot access the file with" + echo "LOV EA hole, it should not cause the system crash." + echo "#####" + + check_mount_and_prep + $LFS mkdir -i 0 $DIR/$tdir/a1 + if [ $OSTCOUNT -gt 2 ]; then + $LFS setstripe -c 3 -i 0 -s 1M $DIR/$tdir/a1 + bcount=513 + else + $LFS setstripe -c 2 -i 0 -s 1M $DIR/$tdir/a1 + bcount=257 + fi + + # 256 blocks on the stripe0. + # 1 block on the stripe1 for 2 OSTs case. + # 256 blocks on the stripe1 for other cases. + # 1 block on the stripe2 if OSTs > 2 + dd if=/dev/zero of=$DIR/$tdir/a1/f0 bs=4096 count=$bcount + dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=4096 count=$bcount + dd if=/dev/zero of=$DIR/$tdir/a1/f2 bs=4096 count=$bcount + + local fid0=$($LFS path2fid $DIR/$tdir/a1/f0) + local fid1=$($LFS path2fid $DIR/$tdir/a1/f1) + local fid2=$($LFS path2fid $DIR/$tdir/a1/f2) + + echo ${fid0} + $LFS getstripe $DIR/$tdir/a1/f0 + echo ${fid1} + $LFS getstripe $DIR/$tdir/a1/f1 + echo ${fid2} + $LFS getstripe $DIR/$tdir/a1/f2 + + if [ $OSTCOUNT -gt 2 ]; then + dd if=/dev/zero of=$DIR/$tdir/a1/f3 bs=4096 count=$bcount + fid3=$($LFS path2fid $DIR/$tdir/a1/f3) + echo ${fid3} + $LFS getstripe $DIR/$tdir/a1/f3 + fi + + cancel_lru_locks osc + + echo "Inject failure..." + echo "To simulate f0 lost MDT-object" + #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616 + do_facet mds1 $LCTL set_param fail_loc=0x1616 + rm -f $DIR/$tdir/a1/f0 + + echo "To simulate f1 lost MDT-object and OST-object0" + #define OBD_FAIL_LFSCK_LOST_SPEOBJ 0x161a + do_facet mds1 $LCTL set_param fail_loc=0x161a + rm -f $DIR/$tdir/a1/f1 + + echo "To simulate f2 lost MDT-object and OST-object1" + do_facet mds1 $LCTL set_param fail_val=1 + rm -f $DIR/$tdir/a1/f2 + + if [ $OSTCOUNT -gt 2 ]; then + echo "To simulate f3 lost MDT-object and OST-object2" + do_facet mds1 $LCTL set_param fail_val=2 + rm -f $DIR/$tdir/a1/f3 + fi + + umount_client $MOUNT + sync + sleep 2 + do_facet mds1 $LCTL set_param fail_loc=0 fail_val=0 + + echo "Inject failure to slow down the LFSCK on OST0" + #define OBD_FAIL_LFSCK_DELAY5 0x161b + do_facet ost1 $LCTL set_param fail_loc=0x161b + + echo "Trigger layout LFSCK on all devices to find out orphan OST-object" + $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!" + + sleep 3 + do_facet ost1 $LCTL set_param fail_loc=0 + + for k in $(seq $MDSCOUNT); do + # The LFSCK status query internal is 30 seconds. For the case + # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough + # time to guarantee the status sync up. + wait_update_facet mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_layout | + awk '/^status/ { print \\\$2 }'" "completed" 32 || + error "(2) MDS${k} is not the expected 'completed'" + done + + for k in $(seq $OSTCOUNT); do + local cur_status=$(do_facet ost${k} $LCTL get_param -n \ + obdfilter.$(facet_svc ost${k}).lfsck_layout | + awk '/^status/ { print $2 }') + [ "$cur_status" == "completed" ] || + error "(3) OST${k} Expect 'completed', but got '$cur_status'" + done + + local repaired=$(do_facet mds1 $LCTL get_param -n \ + mdd.$(facet_svc mds1).lfsck_layout | + awk '/^repaired_orphan/ { print $2 }') + if [ $OSTCOUNT -gt 2 ]; then + [ $repaired -eq 9 ] || + error "(4.1) Expect 9 fixed on mds1, but got: $repaired" + else + [ $repaired -eq 4 ] || + error "(4.2) Expect 4 fixed on mds1, but got: $repaired" + fi + + mount_client $MOUNT || error "(5.0) Fail to start client!" + + LOV_PATTERN_F_HOLE=0x40000000 + + # + # ${fid0}-R-0 is the old f0 + # + local name="$MOUNT/.lustre/lost+found/MDT0000/${fid0}-R-0" + echo "Check $name, which is the old f0" + + $LFS getstripe -v $name || error "(5.1) cannot getstripe on $name" + + local pattern=0x$($LFS getstripe -L $name) + [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] || + error "(5.2) NOT expect pattern flag hole, but got $pattern" + + local stripes=$($LFS getstripe -c $name) + if [ $OSTCOUNT -gt 2 ]; then + [ $stripes -eq 3 ] || + error "(5.3.1) expect the stripe count is 3, but got $stripes" + else + [ $stripes -eq 2 ] || + error "(5.3.2) expect the stripe count is 2, but got $stripes" + fi + + local size=$(stat $name | awk '/Size:/ { print $2 }') + [ $size -eq $((4096 * $bcount)) ] || + error "(5.4) expect the size $((4096 * $bcount)), but got $size" + + cat $name > /dev/null || error "(5.5) cannot read $name" + + echo "dummy" >> $name || error "(5.6) cannot write $name" + + chown $RUNAS_ID:$RUNAS_GID $name || error "(5.7) cannot chown on $name" + + touch $name || error "(5.8) cannot touch $name" + + rm -f $name || error "(5.9) cannot unlink $name" + + # + # ${fid1}-R-0 contains the old f1's stripe1 (and stripe2 if OSTs > 2) + # + name="$MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0" + if [ $OSTCOUNT -gt 2 ]; then + echo "Check $name, it contains the old f1's stripe1 and stripe2" + else + echo "Check $name, it contains the old f1's stripe1" + fi + + $LFS getstripe -v $name || error "(6.1) cannot getstripe on $name" + + pattern=0x$($LFS getstripe -L $name) + [[ $((pattern & LOV_PATTERN_F_HOLE)) -ne 0 ]] || + error "(6.2) expect pattern flag hole, but got $pattern" + + stripes=$($LFS getstripe -c $name) + if [ $OSTCOUNT -gt 2 ]; then + [ $stripes -eq 3 ] || + error "(6.3.1) expect the stripe count is 3, but got $stripes" + else + [ $stripes -eq 2 ] || + error "(6.3.2) expect the stripe count is 2, but got $stripes" + fi + + size=$(stat $name | awk '/Size:/ { print $2 }') + [ $size -eq $((4096 * $bcount)) ] || + error "(6.4) expect the size $((4096 * $bcount)), but got $size" + + cat $name > /dev/null && error "(6.5) normal read $name should fail" + + local failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \ + bs=4096 2>&1 | grep "Input/output error" | wc -l) + + # stripe0 is dummy + [ $failures -eq 256 ] || + error "(6.6) expect 256 IO failures, but get $failures" + + size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }') + [ $size -eq $((4096 * $bcount)) ] || + error "(6.7) expect the size $((4096 * $bcount)), but got $size" + + dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 && + error "(6.8) write to the LOV EA hole should fail" + + dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 seek=300 || + error "(6.9) write to normal stripe should NOT fail" + + echo "foo" >> $name && error "(6.10) append write $name should fail" + + chown $RUNAS_ID:$RUNAS_GID $name || error "(6.11) cannot chown on $name" + + touch $name || error "(6.12) cannot touch $name" + + rm -f $name || error "(6.13) cannot unlink $name" + + # + # ${fid2}-R-0 it contains the old f2's stripe0 (and stripe2 if OSTs > 2) + # + name="$MOUNT/.lustre/lost+found/MDT0000/${fid2}-R-0" + if [ $OSTCOUNT -gt 2 ]; then + echo "Check $name, it contains the old f2's stripe0 and stripe2" + else + echo "Check $name, it contains the old f2's stripe0" + fi + + $LFS getstripe -v $name || error "(7.1) cannot getstripe on $name" + + pattern=0x$($LFS getstripe -L $name) + stripes=$($LFS getstripe -c $name) + size=$(stat $name | awk '/Size:/ { print $2 }') + if [ $OSTCOUNT -gt 2 ]; then + [[ $((pattern & LOV_PATTERN_F_HOLE)) -ne 0 ]] || + error "(7.2.1) expect pattern flag hole, but got $pattern" + + [ $stripes -eq 3 ] || + error "(7.3.1) expect the stripe count is 3, but got $stripes" + + [ $size -eq $((4096 * $bcount)) ] || + error "(7.4.1) expect size $((4096 * $bcount)), but got $size" + + cat $name > /dev/null && + error "(7.5.1) normal read $name should fail" + + failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \ + bs=4096 2>&1 | grep "Input/output error" | wc -l) + # stripe1 is dummy + [ $failures -eq 256 ] || + error "(7.6) expect 256 IO failures, but get $failures" + + size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }') + [ $size -eq $((4096 * $bcount)) ] || + error "(7.7) expect the size $((4096 * $bcount)), but got $size" + + dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 \ + seek=300 && error "(7.8.0) write to the LOV EA hole should fail" + + dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 || + error "(7.8.1) write to normal stripe should NOT fail" + + echo "foo" >> $name && + error "(7.8.3) append write $name should fail" + + chown $RUNAS_ID:$RUNAS_GID $name || + error "(7.9.1) cannot chown on $name" + + touch $name || error "(7.10.1) cannot touch $name" + else + [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] || + error "(7.2.2) NOT expect pattern flag hole, but got $pattern" + + [ $stripes -eq 1 ] || + error "(7.3.2) expect the stripe count is 1, but got $stripes" + + # stripe1 is dummy + [ $size -eq $((4096 * (256 + 0))) ] || + error "(7.4.2) expect the size $((4096 * 256)), but got $size" + + cat $name > /dev/null || error "(7.5.2) cannot read $name" + + echo "dummy" >> $name || error "(7.8.2) cannot write $name" + + chown $RUNAS_ID:$RUNAS_GID $name || + error "(7.9.2) cannot chown on $name" + + touch $name || error "(7.10.2) cannot touch $name" + fi + + rm -f $name || error "(7.11) cannot unlink $name" + + [ $OSTCOUNT -le 2 ] && return + + # + # ${fid3}-R-0 should contains the old f3's stripe0 and stripe1 + # + name="$MOUNT/.lustre/lost+found/MDT0000/${fid3}-R-0" + echo "Check $name, which contains the old f3's stripe0 and stripe1" + + $LFS getstripe -v $name || error "(8.1) cannot getstripe on $name" + + pattern=0x$($LFS getstripe -L $name) + [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] || + error "(8.2) NOT expect pattern flag hole, but got $pattern" + + stripes=$($LFS getstripe -c $name) + # LFSCK does not know the old f3 had 3 stripes. + # It only tries to find as much as possible. + # The stripe count depends on the last stripe's offset. + [ $stripes -eq 2 ] || + error "(8.3) expect the stripe count is 2, but got $stripes" + + size=$(stat $name | awk '/Size:/ { print $2 }') + # stripe2 is lost + [ $size -eq $((4096 * (256 + 256 + 0))) ] || + error "(8.4) expect the size $((4096 * 512)), but got $size" + + cat $name > /dev/null || error "(8.5) cannot read $name" + + echo "dummy" >> $name || error "(8.6) cannot write $name" + + chown $RUNAS_ID:$RUNAS_GID $name || + error "(8.7) cannot chown on $name" + + touch $name || error "(8.8) cannot touch $name" + + rm -f $name || error "(8.9) cannot unlink $name" +} +run_test 20 "Handle the orphan with dummy LOV EA slot properly" + +test_21() { + [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.59) ]] && + skip "ignore the test if MDS is older than 2.5.59" && exit 0 + + check_mount_and_prep + createmany -o $DIR/$tdir/f 100 || error "(0) Fail to create 100 files" + + echo "Start all LFSCK components by default (-s 1)" + do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -s 1 -r || + error "Fail to start LFSCK" + + echo "namespace LFSCK should be in 'scanning-phase1' status" + local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "Expect namespace 'scanning-phase1', but got '$STATUS'" + + echo "layout LFSCK should be in 'scanning-phase1' status" + STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }') + [ "$STATUS" == "scanning-phase1" ] || + error "Expect layout 'scanning-phase1', but got '$STATUS'" + + echo "Stop all LFSCK components by default" + do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 || + error "Fail to stop LFSCK" +} +run_test 21 "run all LFSCK components by default" + $LCTL set_param debug=-lfsck > /dev/null || true # restore MDS/OST size