ONLY=${ONLY:-"$*"}
-#Bug number for excepting test LU-10406
-ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT 31c"
+#Bug number for excepting test
+ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
# use small MDS + OST size to speed formatting time
# do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
MDSSIZE=100000
+[ $(facet_fstype $SINGLEMDS) == zfs ] && MDSSIZE=300000
OSTSIZE=100000
+[ $(facet_fstype ost1) == zfs ] && OSTSIZE=300000
+
# no need too many OSTs, to reduce the format/start/stop overhead
cleanupall
[ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
local D_POS1=$($SHOW_NAMESPACE |
awk '/^latest_start_position/ { print $4 }')
+ echo "Additional debug for 6b"
+ $SHOW_NAMESPACE
if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then
[[ $O_POS0 -lt $O_POS1 ]] ||
error "(7.1) $O_POS1 is not larger than $O_POS0"
error "(6) Speed $SPEED, expected < $MAX_SPEED"
}
- do_facet $SINGLEMDS \
- $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param -n mdd.*.lfsck_speed_limit 0
+ do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL set_param -n obdfilter.*.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS \
"$LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout |
error "(10) Speed $SPEED, expected < $MAX_SPEED"
}
- do_facet $SINGLEMDS \
- $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param -n mdd.*.lfsck_speed_limit 0
+ do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL set_param -n obdfilter.*.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 32 || {
[ "$STATUS" == "scanning-phase1" ] ||
error "(15) Expect 'scanning-phase1', but got '$STATUS'"
- do_facet $SINGLEMDS \
- $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param -n mdd.*.lfsck_speed_limit 0
+ do_nodes $(comma_list $(osts_nodes)) \
+ $LCTL set_param -n obdfilter.*.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 32 || {
dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
cancel_lru_locks osc
+ # created but no setattr or write to the file.
+ mkdir $DIR/$tdir/d1
+ chown $RUNAS_ID:$RUNAS_GID $DIR/$tdir/d1
+ $RUNAS createmany -o $DIR/$tdir/d1/o 100 || error "create failed"
+
echo "Inject failure stub to skip OST-object owner changing"
#define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
run_test 18a "Find out orphan OST-object and repair it (1)"
test_18b() {
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
+
echo "#####"
echo "The target MDT-object is lost. The LFSCK should re-create the"
echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
run_test 18b "Find out orphan OST-object and repair it (2)"
test_18c() {
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
+
echo "#####"
echo "The target MDT-object is lost, and the OST-object FID is missing."
echo "The LFSCK should re-create the MDT-object with new FID under the "
run_test 18d "Find out orphan OST-object and repair it (4)"
test_18e() {
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
+
echo "#####"
echo "The target MDT-object layout EA slot is occpuied by some new"
echo "created OST-object when repair dangling reference case. Such"
run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
test_18g() {
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
+
echo "#####"
echo "The target MDT-object is lost, but related OI mapping is there"
echo "The LFSCK should recreate the lost MDT-object without affected"
test_20a() {
[ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
echo "#####"
echo "The target MDT-object and some of its OST-object are lost."
test_20b() {
[ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
echo "#####"
echo "The target MDT-object and some of its OST-object are lost."
}
run_test 23b "LFSCK can repair dangling name entry (2)"
+cleanup_23c() {
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(10) unexpected status"
+ }
+
+ stop_full_debug_logging
+}
+
test_23c() {
echo "#####"
echo "The objectA has multiple hard links, one of them corresponding"
check_mount_and_prep
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
- $LFS path2fid $DIR/$tdir/d0
+ parent_fid="$($LFS path2fid $DIR/$tdir/d0)"
+ echo "parent_fid=$parent_fid"
createmany -o $DIR/$tdir/d0/t 10 || error "(1.5) Fail to creatmany"
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
- $LFS path2fid $DIR/$tdir/d0/f0
+ f0_fid="$($LFS path2fid $DIR/$tdir/d0/f0)"
+ echo "f0_fid=$f0_fid"
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
- $LFS path2fid $DIR/$tdir/d0/f1
-
- local SEQ0=$($LFS path2fid $DIR/$tdir/d0/f0 | awk -F':' '{print $1}')
- local SEQ1=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $1}')
+ f1_fid="$($LFS path2fid $DIR/$tdir/d0/f1)"
+ echo "f1_fid=$f1_fid"
- if [ "$SEQ0" != "$SEQ1" ]; then
+ if [ "${fid_f0/:.*/}" != "${fid_f1/:.*/}" ]; then
# To guarantee that the f0 and f1 are in the same FID seq
rm -f $DIR/$tdir/d0/f0 ||
error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
echo "dummy" > $DIR/$tdir/d0/f0 ||
error "(3.2) Fail to touch on MDT0"
- $LFS path2fid $DIR/$tdir/d0/f0
+ f0_fid="$($LFS path2fid $DIR/$tdir/d0/f0)"
+ echo "f0_fid=$f0_fid (replaced)"
fi
- local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
- OID=$(printf %d $OID)
+ local oid=$(awk -F':' '{ printf $2 }' <<< $f1_fid)
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
- do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_val=$oid fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
$START_NAMESPACE -r -C ||
error "(7) Fail to start LFSCK for namespace"
- wait_update_facet client "stat $DIR/$tdir/d0/foo |
- awk '/Size/ { print \\\$2 }'" "0" $LTIME || {
+ wait_update_facet client "stat -c%s $DIR/$tdir/d0/foo" "0" $LTIME || {
+ # While unexpected by the test, it is valid for LFSCK to repair
+ # the link to the original object before any data is written.
+ local size=$(stat -c %s $DIR/$tdir/d0/foo)
+
+ if [ "$size" = "6" -a "$(<$DIR/$tdir/d0/foo)" = "dummy" ]; then
+ log "LFSCK repaired file prematurely"
+ cleanup_23c
+ return 0
+ fi
+
stat $DIR/$tdir/d0/foo
$SHOW_NAMESPACE
error "(8) unexpected size"
echo "data" >> $DIR/$tdir/d0/foo || error "(9) Fail to write"
cancel_lru_locks osc
- do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(10) unexpected status"
- }
-
- stop_full_debug_logging
+ cleanup_23c
local repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
test_24() {
[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
echo "#####"
echo "Two MDT-objects back reference the same name entry via their"
run_test 26b "LFSCK can add the missing remote name entry back to the namespace"
test_27a() {
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
+
echo "#####"
echo "The local parent referenced by the MDT-object linkEA is lost."
echo "The namespace LFSCK will re-create the lost parent as orphan."
test_27b() {
[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
echo "#####"
echo "The remote parent referenced by the MDT-object linkEA is lost."
cancel_lru_locks mdc
if [ $MDSCOUNT -ge 2 ]; then
- $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null ||
- error "(3.1) Migrate failure"
+ $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null &&
+ error "(3.1) Migrate should fail"
echo "The object with linkEA overflow should NOT be migrated"
local newfid=$($LFS path2fid $DIR/$tdir/guard/f0)
unlinkmany $DIR/$tdir/foo/ttttttttttt 100 || error "(4) Fail to unlink"
if [ $MDSCOUNT -ge 2 ]; then
- $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null ||
- error "(5.1) Migrate failure"
+ $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null &&
+ error "(5.1) Migrate should fail"
# The overflow timestamp is still there, so migration will fail.
local newfid=$($LFS path2fid $DIR/$tdir/guard/f0)
test_30() {
[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
skip "ldiskfs only test" && return
+ [ -n "$FILESET" ] && skip "Not functional for FILESET set"
echo "#####"
echo "The namespace LFSCK will move the orphans from backend"
mkdir $DIR/$tdir/foo/d0 || error "(3) Fail to mkdir d0"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ local pfid=$($LFS path2fid $DIR/$tdir/foo)
+ local cfid=$($LFS path2fid $DIR/$tdir/foo/d0)
+
touch $DIR/$tdir/foo/d0/f1 || error "(4) Fail to touch f1"
mkdir $DIR/$tdir/foo/d0/d1 || error "(5) Fail to mkdir d1"
ls -ail $MOUNT/.lustre/lost+found/MDT0000/
- cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-*-D-*)
+ local cname=$MOUNT/.lustre/lost+found/MDT0000/${cfid}-${pfid}-D-0
[ ! -z "$cname" ] || error "(20) d0 is not recovered"
- stat ${cname}/d1 || error "(21) d0 is not recovered"
+ stat ${cname}/d1 || error "(21) d1 is not recovered"
stat ${cname}/f1 || error "(22) f1 is not recovered"
}
run_test 30 "LFSCK can recover the orphans from backend /lost+found"
}
run_test 31h "Repair the corrupted shard's name entry"
-test_32()
+test_32a()
{
lfsck_prep 5 5
umount_client $MOUNT
- #define OBD_FAIL_LFSCK_ASSISTANT_DIRECT 0x162d
+ #define OBD_FAIL_LFSCK_ENGINE_DELAY 0x162d
do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x162d
- $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
+ $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
local STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
- error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+ error "(2) Expect 'scanning-phase1', but got '$STATUS'"
echo "stop ost1"
- stop ost1 > /dev/null || error "(4) Fail to stop OST1!"
+ stop ost1 > /dev/null || error "(3) Fail to stop OST1!"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
- sleep 1
+ sleep 4
echo "stop LFSCK"
- $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
+ $STOP_LFSCK || error "(4) Fail to stop LFSCK!"
+
+ start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
+ error "(5) Fail to start ost1"
+}
+run_test 32a "stop LFSCK when some OST failed"
+
+test_32b()
+{
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+
+ lfsck_prep 5 5
+ $LFS mkdir -i 1 $DIR/$tdir/dp ||
+ error "(1) Fail to create $DIR/$tdir/dp"
+ $LFS mkdir -i 0 -c $MDSCOUNT $DIR/$tdir/dp/dc1 ||
+ error "(2) Fail to create $DIR/$tdir/dp/dc1"
+ $LFS mkdir -i 0 -c $MDSCOUNT $DIR/$tdir/dp/dc2 ||
+ error "(3) Fail to create $DIR/$tdir/dp/dc2"
+ umount_client $MOUNT
+
+ #define OBD_FAIL_LFSCK_ENGINE_DELAY 0x162d
+ do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x162d
+ $START_NAMESPACE -r -A || error "(4) Fail to start LFSCK for namespace!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "scanning-phase1" 32 || {
+ $SHOW_NAMESPACE
+ error "(5) unexpected status"
+ }
+
+ echo "stop mds2"
+ stop mds2 > /dev/null || error "(6) Fail to stop MDT2!"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+ sleep 4
+
+ echo "stop LFSCK"
+ $STOP_LFSCK || error "(7) Fail to stop LFSCK!"
+
+ start mds2 $(mdsdevname 2) $MOUNT_OPTS_NOSCRUB > /dev/null ||
+ error "(8) Fail to start MDT2"
}
-run_test 32 "stop LFSCK when some OST failed"
+run_test 32b "stop LFSCK when some MDT failed"
test_33()
{
}
run_test 35 "LFSCK can rebuild the lost agent entry"
+test_36a() {
+ [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
+
+ echo "#####"
+ echo "The target MDT-object's LOV EA corrupted as to lose one of the "
+ echo "mirrors information. The layout LFSCK should rebuild the LOV EA "
+ echo "with the PFID EA of related OST-object(s) belong to the mirror."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
+ error "(0) Fail to create mirror file $DIR/$tdir/f0"
+ $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f1 ||
+ error "(1) Fail to create mirror file $DIR/$tdir/f1"
+ $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f2 ||
+ error "(2) Fail to create mirror file $DIR/$tdir/f2"
+
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=4 ||
+ error "(3) Fail to write $DIR/$tdir/f0"
+ dd if=/dev/zero of=$DIR/$tdir/f1 bs=1M count=4 ||
+ error "(4) Fail to write $DIR/$tdir/f1"
+ dd if=/dev/zero of=$DIR/$tdir/f2 bs=1M count=4 ||
+ error "(5) Fail to write $DIR/$tdir/f2"
+
+ $LFS mirror resync $DIR/$tdir/f0 ||
+ error "(6) Fail to resync $DIR/$tdir/f0"
+ $LFS mirror resync $DIR/$tdir/f1 ||
+ error "(7) Fail to resync $DIR/$tdir/f1"
+ $LFS mirror resync $DIR/$tdir/f2 ||
+ error "(8) Fail to resync $DIR/$tdir/f2"
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ $LFS getstripe $DIR/$tdir/f0 ||
+ error "(9) Fail to getstripe for $DIR/$tdir/f0"
+ $LFS getstripe $DIR/$tdir/f1 ||
+ error "(10) Fail to getstripe for $DIR/$tdir/f1"
+ $LFS getstripe $DIR/$tdir/f2 ||
+ error "(11) Fail to getstripe for $DIR/$tdir/f2"
+
+ echo "Inject failure, to simulate the case of missing one mirror in LOV"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+
+ $LFS mirror split --mirror-id 1 -d $DIR/$tdir/f0 ||
+ error "(12) Fail to split 1st mirror from $DIR/$tdir/f0"
+ $LFS mirror split --mirror-id 2 -d $DIR/$tdir/f1 ||
+ error "(13) Fail to split 2nd mirror from $DIR/$tdir/f1"
+ $LFS mirror split --mirror-id 3 -d $DIR/$tdir/f2 ||
+ error "(14) Fail to split 3rd mirror from $DIR/$tdir/f2"
+
+ sync
+ sleep 2
+ do_facet mds1 $LCTL set_param fail_loc=0
+
+ $LFS getstripe $DIR/$tdir/f0 | grep "lcme_mirror_id:.*1" &&
+ error "(15) The 1st of mirror is not destroyed"
+ $LFS getstripe $DIR/$tdir/f1 | grep "lcme_mirror_id:.*2" &&
+ error "(16) The 2nd of mirror is not destroyed"
+ $LFS getstripe $DIR/$tdir/f2 | grep "lcme_mirror_id:.*3" &&
+ error "(17) The 3rd of mirror is not destroyed"
+
+ local mirrors
+
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f0)
+ [ $mirrors -eq 2 ] || error "(18) $DIR/$tdir/f0 has $mirrors mirrors"
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f1)
+ [ $mirrors -eq 2 ] || error "(19) $DIR/$tdir/f1 has $mirrors mirrors"
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f2)
+ [ $mirrors -eq 2 ] || error "(20) $DIR/$tdir/f2 has $mirrors mirrors"
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(21) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(22) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(23) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 9 ] ||
+ error "(24) Expect 9 fixed on mds1, but got: $repaired"
+
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f0)
+ [ $mirrors -eq 3 ] || error "(25) $DIR/$tdir/f0 has $mirrors mirrors"
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f1)
+ [ $mirrors -eq 3 ] || error "(26) $DIR/$tdir/f1 has $mirrors mirrors"
+ mirrors=$($LFS getstripe -N $DIR/$tdir/f2)
+ [ $mirrors -eq 3 ] || error "(27) $DIR/$tdir/f2 has $mirrors mirrors"
+
+ $LFS getstripe $DIR/$tdir/f0 | grep "lcme_mirror_id:.*1" || {
+ $LFS getstripe $DIR/$tdir/f0
+ error "(28) The 1st of mirror is not recovered"
+ }
+
+ $LFS getstripe $DIR/$tdir/f1 | grep "lcme_mirror_id:.*2" || {
+ $LFS getstripe $DIR/$tdir/f1
+ error "(29) The 2nd of mirror is not recovered"
+ }
+
+ $LFS getstripe $DIR/$tdir/f2 | grep "lcme_mirror_id:.*3" || {
+ $LFS getstripe $DIR/$tdir/f2
+ error "(30) The 3rd of mirror is not recovered"
+ }
+}
+run_test 36a "rebuild LOV EA for mirrored file (1)"
+
+test_36b() {
+ [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
+
+ echo "#####"
+ echo "The mirrored file lost its MDT-object, but relatd OST-objects "
+ echo "are still there. The layout LFSCK should rebuild the LOV EA "
+ echo "with the PFID EA of related OST-object(s) belong to the file. "
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
+ error "(0) Fail to create mirror file $DIR/$tdir/f0"
+
+ local fid=$($LFS path2fid $DIR/$tdir/f0)
+
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=4 ||
+ error "(1) Fail to write $DIR/$tdir/f0"
+ $LFS mirror resync $DIR/$tdir/f0 ||
+ error "(2) Fail to resync $DIR/$tdir/f0"
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ $LFS getstripe $DIR/$tdir/f0 ||
+ error "(3) Fail to getstripe for $DIR/$tdir/f0"
+
+ echo "Inject failure, to simulate the case of missing the MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/f0 || error "(4) Fail to remove $DIR/$tdir/f0"
+
+ sync
+ sleep 2
+ do_facet mds1 $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(5) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(6) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(7) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local count=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $count -eq 9 ] || error "(8) Expect 9 fixed on mds1, but got: $count"
+
+ local name=$MOUNT/.lustre/lost+found/MDT0000/${fid}-R-0
+ count=$($LFS getstripe --mirror-count $name)
+ [ $count -eq 3 ] || error "(9) $DIR/$tdir/f0 has $count mirrors"
+
+ count=$($LFS getstripe --component-count $name)
+ [ $count -eq 6 ] || error "(10) $DIR/$tdir/f0 has $count components"
+
+ $LFS getstripe $name | grep "lcme_mirror_id:.*1" || {
+ $LFS getstripe $name
+ error "(11) The 1st of mirror is not recovered"
+ }
+
+ $LFS getstripe $name | grep "lcme_mirror_id:.*2" || {
+ $LFS getstripe $name
+ error "(12) The 2nd of mirror is not recovered"
+ }
+
+ $LFS getstripe $name | grep "lcme_mirror_id:.*3" || {
+ $LFS getstripe $name
+ error "(13) The 3rd of mirror is not recovered"
+ }
+}
+run_test 36b "rebuild LOV EA for mirrored file (2)"
+
+test_36c() {
+ [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
+
+ echo "#####"
+ echo "The mirrored file has been modified, not resynced yet, then "
+ echo "lost its MDT-object, but relatd OST-objects are still there. "
+ echo "The layout LFSCK should rebuild the LOV EA and relatd status "
+ echo "with the PFID EA of related OST-object(s) belong to the file. "
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ $DIR/$tdir/f0 ||
+ error "(0) Fail to create mirror file $DIR/$tdir/f0"
+
+ local fid=$($LFS path2fid $DIR/$tdir/f0)
+
+ # The 1st dd && resync makes all related OST-objects have been written
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=4 ||
+ error "(1.1) Fail to write $DIR/$tdir/f0"
+ $LFS mirror resync $DIR/$tdir/f0 ||
+ error "(1.2) Fail to resync $DIR/$tdir/f0"
+ # The 2nd dd makes one mirror to be stale
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=4 ||
+ error "(1.3) Fail to write $DIR/$tdir/f0"
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ $LFS getstripe $DIR/$tdir/f0 ||
+ error "(2) Fail to getstripe for $DIR/$tdir/f0"
+
+ local saved_flags1=$($LFS getstripe $DIR/$tdir/f0 | head -n 10 |
+ awk '/lcme_flags/ { print $2 }')
+ local saved_flags2=$($LFS getstripe $DIR/$tdir/f0 | tail -n 10 |
+ awk '/lcme_flags/ { print $2 }')
+
+ echo "Inject failure, to simulate the case of missing the MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/f0 || error "(3) Fail to remove $DIR/$tdir/f0"
+
+ sync
+ sleep 2
+ do_facet mds1 $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(4) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(5) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(6) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local count=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $count -eq 6 ] || error "(7) Expect 9 fixed on mds1, but got: $count"
+
+ local name=$MOUNT/.lustre/lost+found/MDT0000/${fid}-R-0
+ count=$($LFS getstripe --mirror-count $name)
+ [ $count -eq 2 ] || error "(8) $DIR/$tdir/f0 has $count mirrors"
+
+ count=$($LFS getstripe --component-count $name)
+ [ $count -eq 4 ] || error "(9) $DIR/$tdir/f0 has $count components"
+
+ local flags=$($LFS getstripe $name | head -n 10 |
+ awk '/lcme_flags/ { print $2 }')
+ [ "$flags" == "$saved_flags1" ] || {
+ $LFS getstripe $name
+ error "(10) expect flags $saved_flags1, got $flags"
+ }
+
+ flags=$($LFS getstripe $name | tail -n 10 |
+ awk '/lcme_flags/ { print $2 }')
+ [ "$flags" == "$saved_flags2" ] || {
+ $LFS getstripe $name
+ error "(11) expect flags $saved_flags2, got $flags"
+ }
+}
+run_test 36c "rebuild LOV EA for mirrored file (3)"
+
+test_37()
+{
+ local PID
+ local rc
+ local t_dir="$DIR/$tdir/d0"
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $t_dir || error "(2) Fail to mkdir $t_dir on MDT0"
+ multiop_bg_pause $t_dir D_c || { error "multiop failed: $?"; return 1; }
+ PID=$!
+ rmdir $t_dir
+
+ $START_NAMESPACE -r -A || {
+ error "(3) Fail to start LFSCK for namespace!"; kill -USR1 $PID; }
+
+ wait_all_targets_blocked namespace completed 4
+ stat $t_dir && rc=1
+ kill -USR1 $PID
+ return $rc
+}
+run_test 37 "LFSCK must skip a ORPHAN"
+
+
# restore MDS/OST size
MDSSIZE=${SAVED_MDSSIZE}
OSTSIZE=${SAVED_OSTSIZE}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff