require_dsh_mds || exit 0
-MCREATE=${MCREATE:-mcreate}
+LTIME=${LTIME:-120}
+
SAVED_MDSSIZE=${MDSSIZE}
SAVED_OSTSIZE=${OSTSIZE}
SAVED_OSTCOUNT=${OSTCOUNT}
ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
[[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19 20 21"
-[ $(facet_fstype $SINGLEMDS) = "zfs" ] &&
-# bug number for skipped test: LU-4970
- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 14"
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.50) ]] &&
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2d 2e 3 22 23 24 25 26 27 28"
build_test_filter
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(9) unexpected status"
}
$START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(12) unexpected status"
}
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
awk '/^updated_phase1/ { print $2 }')
[ $repaired -eq 1 ] ||
- error "(5) Fail to repair missed FID-in-LMA: $repaired"
+ error "(5) Fail to repair the missing FID-in-LMA: $repaired"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
mount_client $MOUNT || error "(6) Fail to start client!"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
}
-run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
+run_test 1b "LFSCK can find out and repair the missing FID-in-LMA"
test_2a() {
lfsck_prep 1 1
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
# for interop with old server
[ -z "$repaired" ] &&
repaired=$($SHOW_NAMESPACE |
- awk '/^updated_phase1/ { print $2 }')
+ awk '/^updated_phase2/ { print $2 }')
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
}
run_test 2c "LFSCK can find out and remove repeated linkEA entry"
+test_2d()
+{
+ lfsck_prep 1 1
+
+ #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161d
+ touch $DIR/$tdir/dummy
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ umount_client $MOUNT
+ $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(4) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^linkea_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair crashed linkEA: $repaired"
+
+ mount_client $MOUNT || error "(6) Fail to start client!"
+
+ stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
+ error "(7) Fail to stat $DIR/$tdir/dummy"
+
+ local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
+ error "(8) Fail to repair linkEA: $dummyfid $dummyname"
+}
+run_test 2d "LFSCK can recover the missing linkEA entry"
+
+test_2e()
+{
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT1"
+
+ #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
+ $LFS mkdir -i 0 $DIR/$tdir/d0/d1 || error "(2) Fail to mkdir d1 on MDT0"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ $START_NAMESPACE -r -A || error "(3) Fail to start LFSCK for namespace!"
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(4) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^linkea_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair crashed linkEA: $repaired"
+
+ local fid=$($LFS path2fid $DIR/$tdir/d0/d1)
+ local name=$($LFS fid2path $DIR $fid)
+ [ "$name" == "$DIR/$tdir/d0/d1" ] ||
+ error "(6) Fail to repair linkEA: $fid $name"
+}
+run_test 2e "namespace LFSCK can verify remote object linkEA"
+
+test_3()
+{
+ lfsck_prep 4 4
+
+ mkdir $DIR/$tdir/dummy || error "(1) Fail to mkdir"
+ ln $DIR/$tdir/d0/f0 $DIR/$tdir/dummy/f0 || error "(2) Fail to hardlink"
+ ln $DIR/$tdir/d0/f1 $DIR/$tdir/dummy/f1 || error "(3) Fail to hardlink"
+
+ $LFS mkdir -i 0 $DIR/$tdir/edir || error "(4) Fail to mkdir"
+ touch $DIR/$tdir/edir/f0 || error "(5) Fail to touch"
+ touch $DIR/$tdir/edir/f1 || error "(6) Fail to touch"
+
+ #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
+ ln $DIR/$tdir/edir/f0 $DIR/$tdir/edir/w0 || error "(7) Fail to hardlink"
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ ln $DIR/$tdir/edir/f1 $DIR/$tdir/edir/w1 || error "(8) Fail to hardlink"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ $START_NAMESPACE -r || error "(9) Fail to start LFSCK for namespace!"
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(10) unexpected status"
+ }
+
+ local checked=$($SHOW_NAMESPACE |
+ awk '/^checked_phase2/ { print $2 }')
+ [ $checked -ge 4 ] ||
+ error "(11) Fail to check multiple-linked object: $checked"
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^multiple_linked_repaired/ { print $2 }')
+ [ $repaired -ge 2 ] ||
+ error "(12) Fail to repair multiple-linked object: $repaired"
+}
+run_test 3 "LFSCK can verify multiple-linked objects"
+
test_4()
{
[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
$START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^flags/ { print \\\$2 }'" "inconsistent" 6 || {
+ awk '/^flags/ { print \\\$2 }'" "inconsistent" 32 || {
$SHOW_NAMESPACE
error "(5) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(7) unexpected status"
}
$START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^flags/ { print \\\$2 }'" "inconsistent,upgrade" 6 || {
+ awk '/^flags/ { print \\\$2 }'" "inconsistent,upgrade" 32 || {
$SHOW_NAMESPACE
error "(5) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(7) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "failed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "failed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
local POS1=$($SHOW_NAMESPACE |
awk '/^latest_start_position/ { print $2 }' |
tr -d ',')
- [ $POS0 -lt $POS1 ] ||
+ [[ $POS0 -lt $POS1 ]] ||
error "(7) Expect larger than: $POS0, but got $POS1"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(8) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "failed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "failed" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
awk '/^latest_start_position/ { print $4 }')
if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then
- [ $O_POS0 -lt $O_POS1 ] ||
+ [[ $O_POS0 -lt $O_POS1 ]] ||
error "(7.1) $O_POS1 is not larger than $O_POS0"
else
- [ $D_POS0 -lt $D_POS1 ] ||
+ [[ $D_POS0 -lt $D_POS1 ]] ||
error "(7.2) $D_POS1 is not larger than $D_POS0"
fi
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(8) unexpected status"
}
$START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" 32 || {
$SHOW_NAMESPACE
error "(4) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "failed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "failed" 32 || {
$SHOW_NAMESPACE
error "(10) unexpected status"
}
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(14) Fail to start MDS!"
+ local timeout=$(max_recovery_time)
+ local timer=0
+
+ while [ $timer -lt $timeout ]; do
+ STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+ mdt.${MDT_DEV}.recovery_status |
+ awk '/^status/ { print \\\$2 }'")
+ [ "$STATUS" != "RECOVERING" ] && break;
+ sleep 1
+ timer=$((timer + 1))
+ done
+
+ [ $timer != $timeout ] ||
+ error "(14.1) recovery timeout"
+
STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
[ "$STATUS" == "crashed" ] ||
error "(15) Expect 'crashed', but got '$STATUS'"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(19) Fail to start MDS!"
+ timer=0
+ while [ $timer -lt $timeout ]; do
+ STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+ mdt.${MDT_DEV}.recovery_status |
+ awk '/^status/ { print \\\$2 }'")
+ [ "$STATUS" != "RECOVERING" ] && break;
+ sleep 1
+ timer=$((timer + 1))
+ done
+
+ [ $timer != $timeout ] ||
+ error "(19.1) recovery timeout"
+
STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
[ "$STATUS" == "paused" ] ||
error "(20) Expect 'paused', but got '$STATUS'"
$START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" 32 || {
$SHOW_NAMESPACE
error "(22) unexpected status"
}
do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(24) unexpected status"
}
local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
(RUN_TIME1 + RUN_TIME2) * 8 / 10))
- # Account for slow ZFS performance - LU-4934
- [ $SPEED -gt $MIN_SPEED ] || [ $(facet_fstype $SINGLEMDS) -eq zfs ] ||
- error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
+ [ $SPEED -gt $MIN_SPEED ] || {
+ if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+ error_ignore LU-5624 \
+ "(5.1) Got speed $SPEED, expected more than $MIN_SPEED"
+ else
+ error \
+ "(5.2) Got speed $SPEED, expected more than $MIN_SPEED"
+ fi
+ }
# MAX_MARGIN = 1.2 = 12 / 10
MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
(RUN_TIME1 + RUN_TIME2) * 8 / 10))
- [ $SPEED -gt $MIN_SPEED ] ||[ $(facet_fstype $SINGLEMDS) -eq zfs ] ||
- error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
+ [ $SPEED -gt $MIN_SPEED ] || {
+ if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+ error_ignore LU-5624 \
+ "(9.1) Got speed $SPEED, expected more than $MIN_SPEED"
+ else
+ error \
+ "(9.2) Got speed $SPEED, expected more than $MIN_SPEED"
+ fi
+ }
# MAX_MARGIN = 1.2 = 12 / 10
MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(11) unexpected status"
}
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_NAMESPACE
error "(16) unexpected status"
}
wait_update_facet ost1 "$LCTL get_param -n \
obdfilter.${OST_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT_ON_OST
error "(6) unexpected status"
}
wait_update_facet ost1 "$LCTL get_param -n \
obdfilter.${OST_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT_ON_OST
error "(6) unexpected status"
}
test_12() {
[ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for test_12" && exit 0
+ skip "We need at least 2 MDSes for test_12" && return
check_mount_and_prep
for k in $(seq $MDSCOUNT); do
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(2) unexpected status"
}
test_14() {
echo "#####"
echo "The OST-object referenced by the MDT-object should be there;"
- echo "otherwise, the LFSCK should re-create the missed OST-object."
+ echo "otherwise, the LFSCK should re-create the missing OST-object."
echo "#####"
check_mount_and_prep
echo "Inject failure stub to simulate dangling referenced MDT-object"
#define OBD_FAIL_LFSCK_DANGLING 0x1610
do_facet ost1 $LCTL set_param fail_loc=0x1610
- createmany -o $DIR/$tdir/f $((count + 32))
+ createmany -o $DIR/$tdir/f $((count + 31))
+ touch $DIR/$tdir/guard
do_facet ost1 $LCTL set_param fail_loc=0
+ start_full_debug_logging
+
# exhaust other pre-created dangling cases
count=$(precreated_ost_obj_count 0 0)
createmany -o $DIR/$tdir/a $count ||
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(3) unexpected status"
}
[ $repaired -ge 32 ] ||
error "(4) Fail to repair dangling reference: $repaired"
- echo "'ls' should fail because it will not repair dangling by default"
- ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail."
+ echo "'stat' should fail because of not repair dangling by default"
+ stat $DIR/$tdir/guard > /dev/null 2>&1 && error "(5) stat should fail"
echo "Trigger layout LFSCK to repair dangling reference"
$START_LAYOUT -r -c || error "(6) Fail to start LFSCK for layout!"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(7) unexpected status"
}
+ # There may be some async LFSCK updates in processing, wait for
+ # a while until the target reparation has been done. LU-4970.
+
+ echo "'stat' should success after layout LFSCK repairing"
+ wait_update_facet client "stat $DIR/$tdir/guard |
+ awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ stat $DIR/$tdir/guard
+ $SHOW_LAYOUT
+ error "(8) unexpected size"
+ }
+
repaired=$($SHOW_LAYOUT |
awk '/^repaired_dangling/ { print $2 }')
[ $repaired -ge 32 ] ||
- error "(8) Fail to repair dangling reference: $repaired"
+ error "(9) Fail to repair dangling reference: $repaired"
- echo "'ls' should success after layout LFSCK repairing"
- ls -ail $DIR/$tdir > /dev/null || error "(9) ls should success."
+ stop_full_debug_logging
}
run_test 14 "LFSCK can repair MDT-object with dangling reference"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(2) unexpected status"
}
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(2) unexpected status"
}
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(2) unexpected status"
}
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
$SHOW_LAYOUT
error "(3) unexpected status"
}
test_18a() {
echo "#####"
echo "The target MDT-object is there, but related stripe information"
- echo "is lost or partly lost. The LFSCK should regenerate the missed"
+ echo "is lost or partly lost. The LFSCK should regenerate the missing"
echo "layout EA entries."
echo "#####"
# time to guarantee the status sync up.
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
error "(4) MDS${k} is not the expected 'completed'"
done
# time to guarantee the status sync up.
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
error "(2) MDS${k} is not the expected 'completed'"
done
if [ $MDSCOUNT -ge 2 ]; then
$LFS mkdir -i 1 $DIR/$tdir/a2
- $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
- do_facet ost2 $LCTL set_param fail_loc=0x1617
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a2
dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
$LFS getstripe $DIR/$tdir/a2/f2
fi
# time to guarantee the status sync up.
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
error "(2) MDS${k} is not the expected 'completed'"
done
done
if [ $MDSCOUNT -ge 2 ]; then
- expected=3
+ expected=2
else
expected=1
fi
error "(5) Expect 0 fixed on mds2, but got: $repaired"
fi
- echo "There should be some stub under .lustre/lost+found/MDT0001/"
- ls -ail $MOUNT/.lustre/lost+found/MDT0001/*-N-0 &&
- error "(6) .lustre/lost+found/MDT0001/ should be empty"
+ ls -ail $MOUNT/.lustre/lost+found/
+
+ echo "There should NOT be some stub under .lustre/lost+found/MDT0001/"
+ if [ -d $MOUNT/.lustre/lost+found/MDT0001 ]; then
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0001/ -name *-N-*)
+ [ -z "$cname" ] ||
+ error "(6) .lustre/lost+found/MDT0001/ should be empty"
+ fi
echo "There should be some stub under .lustre/lost+found/MDT0000/"
- ls -ail $MOUNT/.lustre/lost+found/MDT0000/*-N-0 ||
- error "(7) .lustre/lost+found/MDT0000/ should not be empty"
+ [ -d $MOUNT/.lustre/lost+found/MDT0000 ] ||
+ error "(7) $MOUNT/.lustre/lost+found/MDT0000/ should be there"
+
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-N-*)
+ [ ! -z "$cname" ] ||
+ error "(8) .lustre/lost+found/MDT0000/ should not be empty"
}
run_test 18c "Find out orphan OST-object and repair it (3)"
wait_update_facet mds1 "$LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" $LTIME ||
error "(3.0) MDS1 is not the expected 'scanning-phase2'"
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
# time to guarantee the status sync up.
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
error "(3) MDS${k} is not the expected 'completed'"
done
wait_update_facet mds1 "$LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" $LTIME ||
error "(3) MDS1 is not the expected 'scanning-phase2'"
# to guarantee all updates are synced.
# time to guarantee the status sync up.
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
error "(4) MDS${k} is not the expected 'completed'"
done
error "(6) Expect 1 orphan has been fixed, but got: $repaired"
echo "There should be stub file under .lustre/lost+found/MDT0000/"
- local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/*-C-0)
- [ ! -z $name ] ||
- error "(7) .lustre/lost+found/MDT0000/ should not be empty"
+ [ -d $MOUNT/.lustre/lost+found/MDT0000 ] ||
+ error "(7) $MOUNT/.lustre/lost+found/MDT0000/ should be there"
+
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-C-*)
+ [ ! -z "$cname" ] ||
+ error "(8) .lustre/lost+found/MDT0000/ should not be empty"
echo "The stub file should keep the original f2 data"
cur_size=$(ls -il $cname | awk '{ print $6 }')
[ "$cur_size" == "$saved_size" ] ||
- error "(8) Expect file2 size $saved_size, but got $cur_size"
+ error "(9) Expect file2 size $saved_size, but got $cur_size"
cat $cname
$LFS path2fid $cname
}
run_test 18e "Find out orphan OST-object and repair it (5)"
+test_18f() {
+ [ $OSTCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 OSTs" && return
+
+ echo "#####"
+ echo "The target MDT-object is lost. The LFSCK should re-create the"
+ echo "MDT-object under .lustre/lost+found/MDTxxxx. If some OST fail"
+ echo "to verify some OST-object(s) during the first stage-scanning,"
+ echo "the LFSCK should skip orphan OST-objects for such OST. Others"
+ echo "should not be affected."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/guard bs=1M count=2
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
+ $LFS mkdir -i 0 $DIR/$tdir/a2
+ $LFS setstripe -c 2 -i 0 -s 1M $DIR/$tdir/a2
+ dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
+ $LFS getstripe $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a2/f2
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS mkdir -i 1 $DIR/$tdir/a3
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a3
+ dd if=/dev/zero of=$DIR/$tdir/a3/guard bs=1M count=2
+ dd if=/dev/zero of=$DIR/$tdir/a3/f3 bs=1M count=2
+ $LFS mkdir -i 1 $DIR/$tdir/a4
+ $LFS setstripe -c 2 -i 0 -s 1M $DIR/$tdir/a4
+ dd if=/dev/zero of=$DIR/$tdir/a4/f4 bs=1M count=2
+ $LFS getstripe $DIR/$tdir/a3/f3
+ $LFS getstripe $DIR/$tdir/a4/f4
+ fi
+
+ cancel_lru_locks osc
+
+ echo "Inject failure, to simulate the case of missing the MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a1/f1
+ rm -f $DIR/$tdir/a2/f2
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a3/f3
+ rm -f $DIR/$tdir/a4/f4
+ fi
+
+ sync
+ sleep 2
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0
+ fi
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Inject failure, to simulate the OST0 fail to handle"
+ echo "MDT0 LFSCK request during the first-stage scanning."
+ #define OBD_FAIL_LFSCK_BAD_NETWORK 0x161c
+ do_facet mds1 $LCTL set_param fail_loc=0x161c fail_val=0
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "partial" $LTIME ||
+ error "(2) MDS${k} is not the expected 'partial'"
+ done
+
+ wait_update_facet ost1 "$LCTL get_param -n \
+ obdfilter.$(facet_svc ost1).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "partial" $LTIME || {
+ error "(3) OST1 is not the expected 'partial'"
+ }
+
+ wait_update_facet ost2 "$LCTL get_param -n \
+ obdfilter.$(facet_svc ost2).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME || {
+ error "(4) OST2 is not the expected 'completed'"
+ }
+
+ do_facet mds1 $LCTL set_param fail_loc=0 fail_val=0
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Expect 1 fixed on mds{1}, but got: $repaired"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(6) Expect 1 fixed on mds{2}, but got: $repaired"
+ fi
+
+ echo "Trigger layout LFSCK on all devices again to cleanup"
+ $START_LAYOUT -r -o || error "(7) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
+ error "(8) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(9) OST${k} Expect 'completed', but got '$cur_status'"
+
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 2 ] ||
+ error "(10) Expect 2 fixed on mds{1}, but got: $repaired"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 2 ] ||
+ error "(11) Expect 2 fixed on mds{2}, but got: $repaired"
+ fi
+}
+run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
+
test_19a() {
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
}
run_test 20 "Handle the orphan with dummy LOV EA slot properly"
+test_21() {
+ [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.59) ]] &&
+ skip "ignore the test if MDS is older than 2.5.59" && return
+
+ check_mount_and_prep
+ createmany -o $DIR/$tdir/f 100 || error "(0) Fail to create 100 files"
+
+ echo "Start all LFSCK components by default (-s 1)"
+ do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -s 1 -r ||
+ error "Fail to start LFSCK"
+
+ echo "namespace LFSCK should be in 'scanning-phase1' status"
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "Expect namespace 'scanning-phase1', but got '$STATUS'"
+
+ echo "layout LFSCK should be in 'scanning-phase1' status"
+ STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "Expect layout 'scanning-phase1', but got '$STATUS'"
+
+ echo "Stop all LFSCK components by default"
+ do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 ||
+ error "Fail to stop LFSCK"
+}
+run_test 21 "run all LFSCK components by default"
+
+test_22a() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "The parent_A references the child directory via some name entry,"
+ echo "but the child directory back references another parent_B via its"
+ echo "".." name entry. The parent_A does not exist. Then the namesapce"
+ echo "LFSCK will repair the child directory's ".." name entry."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/guard || error "(1) Fail to mkdir on MDT1"
+ $LFS mkdir -i 1 $DIR/$tdir/foo || error "(2) Fail to mkdir on MDT1"
+
+ echo "Inject failure stub on MDT0 to simulate bad dotdot name entry"
+ echo "The dummy's dotdot name entry references the guard."
+ #define OBD_FAIL_LFSCK_BAD_PARENT 0x161e
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161e
+ $LFS mkdir -i 0 $DIR/$tdir/foo/dummy ||
+ error "(3) Fail to mkdir on MDT0"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ rmdir $DIR/$tdir/guard || error "(4) Fail to rmdir $DIR/$tdir/guard"
+
+ echo "Trigger namespace LFSCK to repair unmatched pairs"
+ $START_NAMESPACE -A -r ||
+ error "(5) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(6) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^unmatched_pairs_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(7) Fail to repair unmatched pairs: $repaired"
+
+ echo "'ls' should success after namespace LFSCK repairing"
+ ls -ail $DIR/$tdir/foo/dummy > /dev/null ||
+ error "(8) ls should success."
+}
+run_test 22a "LFSCK can repair unmatched pairs (1)"
+
+test_22b() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "The parent_A references the child directory via the name entry_B,"
+ echo "but the child directory back references another parent_C via its"
+ echo "".." name entry. The parent_C exists, but there is no the name"
+ echo "entry_B under the parent_B. Then the namesapce LFSCK will repair"
+ echo "the child directory's ".." name entry and its linkEA."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/guard || error "(1) Fail to mkdir on MDT1"
+ $LFS mkdir -i 1 $DIR/$tdir/foo || error "(2) Fail to mkdir on MDT1"
+
+ echo "Inject failure stub on MDT0 to simulate bad dotdot name entry"
+ echo "and bad linkEA. The dummy's dotdot name entry references the"
+ echo "guard. The dummy's linkEA references n non-exist name entry."
+ #define OBD_FAIL_LFSCK_BAD_PARENT2 0x161f
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161f
+ $LFS mkdir -i 0 $DIR/$tdir/foo/dummy ||
+ error "(3) Fail to mkdir on MDT0"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ local dummyfid=$($LFS path2fid $DIR/$tdir/foo/dummy)
+ echo "fid2path should NOT work on the dummy's FID $dummyfid"
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" != "$DIR/$tdir/foo/dummy" ] ||
+ error "(4) fid2path works unexpectedly."
+
+ echo "Trigger namespace LFSCK to repair unmatched pairs"
+ $START_NAMESPACE -A -r ||
+ error "(5) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(6) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^unmatched_pairs_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(7) Fail to repair unmatched pairs: $repaired"
+
+ echo "fid2path should work on the dummy's FID $dummyfid after LFSCK"
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" == "$DIR/$tdir/foo/dummy" ] ||
+ error "(8) fid2path does not work"
+}
+run_test 22b "LFSCK can repair unmatched pairs (2)"
+
+test_23a() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "The name entry is there, but the MDT-object for such name "
+ echo "entry does not exist. The namespace LFSCK should find out "
+ echo "and repair the inconsistency as required."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ $LFS mkdir -i 1 $DIR/$tdir/d0/d1 || error "(2) Fail to mkdir d1 on MDT1"
+
+ echo "Inject failure stub on MDT1 to simulate dangling name entry"
+ #define OBD_FAIL_LFSCK_DANGLING2 0x1620
+ do_facet mds2 $LCTL set_param fail_loc=0x1620
+ rmdir $DIR/$tdir/d0/d1 || error "(3) Fail to rmdir d1"
+ do_facet mds2 $LCTL set_param fail_loc=0
+
+ echo "'ls' should fail because of dangling name entry"
+ ls -ail $DIR/$tdir/d0/d1 > /dev/null 2>&1 && error "(4) ls should fail."
+
+ echo "Trigger namespace LFSCK to find out dangling name entry"
+ $START_NAMESPACE -A -r ||
+ error "(5) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(6) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^dangling_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(7) Fail to repair dangling name entry: $repaired"
+
+ echo "'ls' should fail because not re-create MDT-object by default"
+ ls -ail $DIR/$tdir/d0/d1 > /dev/null 2>&1 && error "(8) ls should fail."
+
+ echo "Trigger namespace LFSCK again to repair dangling name entry"
+ $START_NAMESPACE -A -r -C ||
+ error "(9) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(10) unexpected status"
+ }
+
+ repaired=$($SHOW_NAMESPACE |
+ awk '/^dangling_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(11) Fail to repair dangling name entry: $repaired"
+
+ echo "'ls' should success after namespace LFSCK repairing"
+ ls -ail $DIR/$tdir/d0/d1 > /dev/null || error "(12) ls should success."
+}
+run_test 23a "LFSCK can repair dangling name entry (1)"
+
+test_23b() {
+ echo "#####"
+ echo "The objectA has multiple hard links, one of them corresponding"
+ echo "to the name entry_B. But there is something wrong for the name"
+ echo "entry_B and cause entry_B to references non-exist object_C."
+ echo "In the first-stage scanning, the LFSCK will think the entry_B"
+ echo "as dangling, and re-create the lost object_C. When the LFSCK"
+ echo "comes to the second-stage scanning, it will find that the"
+ echo "former re-creating object_C is not proper, and will try to"
+ echo "replace the object_C with the real object_A."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+
+ echo "Inject failure stub on MDT0 to simulate dangling name entry"
+ #define OBD_FAIL_LFSCK_DANGLING3 0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
+
+ echo "'ls' should fail because of dangling name entry"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(6) ls should fail."
+
+ echo "Trigger namespace LFSCK to find out dangling name entry"
+ $START_NAMESPACE -r -C ||
+ error "(7) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(8) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^dangling_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(9) Fail to repair dangling name entry: $repaired"
+
+ repaired=$($SHOW_NAMESPACE |
+ awk '/^multiple_linked_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(10) Fail to drop the former created object: $repaired"
+
+ local data=$(cat $DIR/$tdir/d0/foo)
+ [ "$data" == "dummy" ] ||
+ error "(11) The $DIR/$tdir/d0/foo is not recovered: $data"
+}
+run_test 23b "LFSCK can repair dangling name entry (2)"
+
+test_23c() {
+ echo "#####"
+ echo "The objectA has multiple hard links, one of them corresponding"
+ echo "to the name entry_B. But there is something wrong for the name"
+ echo "entry_B and cause entry_B to references non-exist object_C."
+ echo "In the first-stage scanning, the LFSCK will think the entry_B"
+ echo "as dangling, and re-create the lost object_C. And then others"
+ echo "modified the re-created object_C. When the LFSCK comes to the"
+ echo "second-stage scanning, it will find that the former re-creating"
+ echo "object_C maybe wrong and try to replace the object_C with the"
+ echo "real object_A. But because object_C has been modified, so the"
+ echo "LFSCK cannot replace it."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+
+ echo "Inject failure stub on MDT0 to simulate dangling name entry"
+ #define OBD_FAIL_LFSCK_DANGLING3 0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
+
+ echo "'ls' should fail because of dangling name entry"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(6) ls should fail."
+
+ #define OBD_FAIL_LFSCK_DELAY3 0x1602
+ do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
+
+ echo "Trigger namespace LFSCK to find out dangling name entry"
+ $START_NAMESPACE -r -C ||
+ error "(7) Fail to start LFSCK for namespace"
+
+ wait_update_facet client "stat $DIR/$tdir/d0/foo |
+ awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ stat $DIR/$tdir/guard
+ $SHOW_NAMESPACE
+ error "(8) unexpected size"
+ }
+
+ echo "data" >> $DIR/$tdir/d0/foo || error "(9) Fail to write"
+ cancel_lru_locks osc
+
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(10) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^dangling_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(11) Fail to repair dangling name entry: $repaired"
+
+ local data=$(cat $DIR/$tdir/d0/foo)
+ [ "$data" != "dummy" ] ||
+ error "(12) The $DIR/$tdir/d0/foo should not be recovered"
+}
+run_test 23c "LFSCK can repair dangling name entry (3)"
+
+test_24() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "Two MDT-objects back reference the same name entry via their"
+ echo "each own linkEA entry, but the name entry only references one"
+ echo "MDT-object. The namespace LFSCK will remove the linkEA entry"
+ echo "for the MDT-object that is not recognized. If such MDT-object"
+ echo "has no other linkEA entry after the removing, then the LFSCK"
+ echo "will add it as orphan under the .lustre/lost+found/MDTxxxx/."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+
+ mkdir $DIR/$tdir/d0/guard || error "(1) Fail to mkdir guard"
+ $LFS path2fid $DIR/$tdir/d0/guard
+
+ mkdir $DIR/$tdir/d0/dummy || error "(2) Fail to mkdir dummy"
+ $LFS path2fid $DIR/$tdir/d0/dummy
+ local pfid=$($LFS path2fid $DIR/$tdir/d0/dummy)
+
+ touch $DIR/$tdir/d0/guard/foo ||
+ error "(3) Fail to touch $DIR/$tdir/d0/guard/foo"
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "the $DIR/$tdir/d0/dummy/foo has the 'bad' linkEA entry"
+ echo "that references $DIR/$tdir/d0/guard/foo."
+ echo "Then remove the name entry $DIR/$tdir/d0/dummy/foo."
+ echo "So the MDT-object $DIR/$tdir/d0/dummy/foo will be left"
+ echo "there with the same linkEA entry as another MDT-object"
+ echo "$DIR/$tdir/d0/guard/foo has"
+
+ #define OBD_FAIL_LFSCK_MUL_REF 0x1622
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1622
+ $LFS mkdir -i 0 $DIR/$tdir/d0/dummy/foo ||
+ error "(4) Fail to mkdir $DIR/$tdir/d0/dummy/foo"
+ local cfid=$($LFS path2fid $DIR/$tdir/d0/dummy/foo)
+ rmdir $DIR/$tdir/d0/dummy/foo ||
+ error "(5) Fail to remove $DIR/$tdir/d0/dummy/foo name entry"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "stat $DIR/$tdir/d0/dummy/foo should fail"
+ stat $DIR/$tdir/d0/dummy/foo > /dev/null 2>&1 &&
+ error "(6) stat successfully unexpectedly"
+
+ echo "Trigger namespace LFSCK to repair multiple-referenced name entry"
+ $START_NAMESPACE -A -r ||
+ error "(7) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(8) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^multiple_referenced_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(9) Fail to repair multiple referenced name entry: $repaired"
+
+ echo "There should be an orphan under .lustre/lost+found/MDT0000/"
+ [ -d $MOUNT/.lustre/lost+found/MDT0000 ] ||
+ error "(10) $MOUNT/.lustre/lost+found/MDT0000/ should be there"
+
+ local cname="$cfid-$pfid-D-0"
+ ls -ail $MOUNT/.lustre/lost+found/MDT0000/$cname ||
+ error "(11) .lustre/lost+found/MDT0000/ should not be empty"
+}
+run_test 24 "LFSCK can repair multiple-referenced name entry"
+
+test_25() {
+ [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
+ skip "Only support to inject failure on ldiskfs" && return
+
+ echo "#####"
+ echo "The file type in the name entry does not match the file type"
+ echo "claimed by the referenced object. Then the LFSCK will update"
+ echo "the file type in the name entry."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "the file type stored in the name entry is wrong."
+
+ #define OBD_FAIL_LFSCK_BAD_TYPE 0x1623
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1623
+ touch $DIR/$tdir/d0/foo || error "(2) Fail to touch $DIR/$tdir/d0/foo"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "Trigger namespace LFSCK to repair bad file type in the name entry"
+ $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(4) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^bad_file_type_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair bad file type in name entry: $repaired"
+
+ ls -ail $DIR/$tdir/d0 || error "(6) Fail to 'ls' the $DIR/$tdir/d0"
+}
+run_test 25 "LFSCK can repair bad file type in the name entry"
+
+test_26a() {
+ echo "#####"
+ echo "The local name entry back referenced by the MDT-object is lost."
+ echo "The namespace LFSCK will add the missing local name entry back"
+ echo "to the normal namespace."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+ touch $DIR/$tdir/d0/foo || error "(2) Fail to create foo"
+ local foofid=$($LFS path2fid $DIR/$tdir/d0/foo)
+
+ ln $DIR/$tdir/d0/foo $DIR/$tdir/d0/dummy ||
+ error "(3) Fail to hard link to $DIR/$tdir/d0/foo"
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "foo's name entry will be removed, but the foo's object"
+ echo "and its linkEA are kept in the system."
+
+ #define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1624
+ rm -f $DIR/$tdir/d0/foo || error "(4) Fail to unlink $DIR/$tdir/d0/foo"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(5) 'ls' should fail"
+
+ echo "Trigger namespace LFSCK to repair the missing remote name entry"
+ $START_NAMESPACE -r -A ||
+ error "(6) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(7) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(8) Fail to repair lost dirent: $repaired"
+
+ ls -ail $DIR/$tdir/d0/foo ||
+ error "(9) Fail to 'ls' $DIR/$tdir/d0/foo"
+
+ local foofid2=$($LFS path2fid $DIR/$tdir/d0/foo)
+ [ "$foofid" == "$foofid2" ] ||
+ error "(10) foo's FID changed: $foofid, $foofid2"
+}
+run_test 26a "LFSCK can add the missing local name entry back to the namespace"
+
+test_26b() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "The remote name entry back referenced by the MDT-object is lost."
+ echo "The namespace LFSCK will add the missing remote name entry back"
+ echo "to the normal namespace."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+ $LFS mkdir -i 0 $DIR/$tdir/d0/foo || error "(2) Fail to mkdir foo"
+ local foofid=$($LFS path2fid $DIR/$tdir/d0/foo)
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "foo's name entry will be removed, but the foo's object"
+ echo "and its linkEA are kept in the system."
+
+ #define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1624
+ rmdir $DIR/$tdir/d0/foo || error "(3) Fail to rmdir $DIR/$tdir/d0/foo"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(4) 'ls' should fail"
+
+ echo "Trigger namespace LFSCK to repair the missing remote name entry"
+ $START_NAMESPACE -r -A ||
+ error "(5) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(6) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(7) Fail to repair lost dirent: $repaired"
+
+ ls -ail $DIR/$tdir/d0/foo ||
+ error "(8) Fail to 'ls' $DIR/$tdir/d0/foo"
+
+ local foofid2=$($LFS path2fid $DIR/$tdir/d0/foo)
+ [ "$foofid" == "$foofid2" ] ||
+ error "(9) foo's FID changed: $foofid, $foofid2"
+}
+run_test 26b "LFSCK can add the missing remote name entry back to the namespace"
+
+test_27a() {
+ echo "#####"
+ echo "The local parent referenced by the MDT-object linkEA is lost."
+ echo "The namespace LFSCK will re-create the lost parent as orphan."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+ touch $DIR/$tdir/d0/foo || error "(2) Fail to create foo"
+ ln $DIR/$tdir/d0/foo $DIR/$tdir/d0/dummy ||
+ error "(3) Fail to hard link to $DIR/$tdir/d0/foo"
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "foo's name entry will be removed, but the foo's object"
+ echo "and its linkEA are kept in the system. And then remove"
+ echo "another hard link and the parent directory."
+
+ #define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1624
+ rm -f $DIR/$tdir/d0/foo ||
+ error "(4) Fail to unlink $DIR/$tdir/d0/foo"
+ rm -f $DIR/$tdir/d0/dummy ||
+ error "(5) Fail to unlink $DIR/$tdir/d0/dummy"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ rm -rf $DIR/$tdir/d0 || error "(5) Fail to unlink the dir d0"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(6) 'ls' should fail"
+
+ echo "Trigger namespace LFSCK to repair the lost parent"
+ $START_NAMESPACE -r -A ||
+ error "(6) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(7) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(8) Fail to repair lost dirent: $repaired"
+
+ echo "There should be an orphan under .lustre/lost+found/MDT0000/"
+ [ -d $MOUNT/.lustre/lost+found/MDT0000 ] ||
+ error "(9) $MOUNT/.lustre/lost+found/MDT0000/ should be there"
+
+ ls -ail $MOUNT/.lustre/lost+found/MDT0000/
+
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-P-*)
+ [ ! -z "$cname" ] ||
+ error "(10) .lustre/lost+found/MDT0000/ should not be empty"
+}
+run_test 27a "LFSCK can recreate the lost local parent directory as orphan"
+
+test_27b() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "The remote parent referenced by the MDT-object linkEA is lost."
+ echo "The namespace LFSCK will re-create the lost parent as orphan."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS mkdir -i 1 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
+ $LFS mkdir -i 0 $DIR/$tdir/d0/foo || error "(2) Fail to mkdir foo"
+
+ $LFS path2fid $DIR/$tdir/d0
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "foo's name entry will be removed, but the foo's object"
+ echo "and its linkEA are kept in the system. And then remove"
+ echo "the parent directory."
+
+ #define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1624
+ rmdir $DIR/$tdir/d0/foo || error "(3) Fail to rmdir $DIR/$tdir/d0/foo"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ rmdir $DIR/$tdir/d0 || error "(4) Fail to unlink the dir d0"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(5) 'ls' should fail"
+
+ echo "Trigger namespace LFSCK to repair the missing remote name entry"
+ $START_NAMESPACE -r -A ||
+ error "(6) Fail to start LFSCK for namespace"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ $SHOW_NAMESPACE
+ error "(7) unexpected status"
+ }
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(8) Fail to repair lost dirent: $repaired"
+
+ ls -ail $MOUNT/.lustre/lost+found/
+
+ echo "There should be an orphan under .lustre/lost+found/MDT0001/"
+ [ -d $MOUNT/.lustre/lost+found/MDT0001 ] ||
+ error "(9) $MOUNT/.lustre/lost+found/MDT0001/ should be there"
+
+ ls -ail $MOUNT/.lustre/lost+found/MDT0001/
+
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0001/ -name *-P-*)
+ [ ! -z "$cname" ] ||
+ error "(10) .lustre/lost+found/MDT0001/ should not be empty"
+}
+run_test 27b "LFSCK can recreate the lost remote parent directory as orphan"
+
+test_28() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "The target name entry is lost. The LFSCK should insert the"
+ echo "orphan MDT-object under .lustre/lost+found/MDTxxxx. But if"
+ echo "the MDT (on which the orphan MDT-object resides) has ever"
+ echo "failed to respond some name entry verification durin the"
+ echo "first stage-scanning, then the LFSCK should skip to handle"
+ echo "orphan MDT-object on this MDT. But other MDTs should not"
+ echo "be affected."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/d1
+ $LFS mkdir -i 1 $DIR/$tdir/d1/a1
+ $LFS mkdir -i 1 $DIR/$tdir/d1/a2
+
+ $LFS mkdir -i 1 $DIR/$tdir/d2
+ $LFS mkdir -i 0 $DIR/$tdir/d2/a1
+ $LFS mkdir -i 0 $DIR/$tdir/d2/a2
+
+ echo "Inject failure stub on MDT0 to simulate the case that"
+ echo "d1/a1's name entry will be removed, but the d1/a1's object"
+ echo "and its linkEA are kept in the system. And the case that"
+ echo "d2/a2's name entry will be removed, but the d2/a2's object"
+ echo "and its linkEA are kept in the system."
+
+ #define OBD_FAIL_LFSCK_NO_NAMEENTRY 0x1624
+ do_facet mds1 $LCTL set_param fail_loc=0x1624
+ do_facet mds2 $LCTL set_param fail_loc=0x1624
+ rmdir $DIR/$tdir/d1/a1 || error "(1) Fail to rmdir $DIR/$tdir/d1/a1"
+ rmdir $DIR/$tdir/d2/a2 || error "(2) Fail to rmdir $DIR/$tdir/d2/a2"
+ do_facet mds1 $LCTL set_param fail_loc=0
+ do_facet mds2 $LCTL set_param fail_loc=0
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Inject failure, to simulate the MDT0 fail to handle"
+ echo "MDT1 LFSCK request during the first-stage scanning."
+ #define OBD_FAIL_LFSCK_BAD_NETWORK 0x161c
+ do_facet mds2 $LCTL set_param fail_loc=0x161c fail_val=0
+
+ echo "Trigger namespace LFSCK on all devices to find out orphan object"
+ $START_NAMESPACE -r -A ||
+ error "(3) Fail to start LFSCK for namespace"
+
+ wait_update_facet mds1 "$LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "partial" 32 || {
+ error "(4) mds1 is not the expected 'partial'"
+ }
+
+ wait_update_facet mds2 "$LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+ error "(5) mds2 is not the expected 'completed'"
+ }
+
+ do_facet mds2 $LCTL set_param fail_loc=0 fail_val=0
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_namespace |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(6) Expect 0 fixed on mds1, but got: $repaired"
+
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_namespace |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(7) Expect 1 fixed on mds2, but got: $repaired"
+
+ echo "Trigger namespace LFSCK on all devices again to cleanup"
+ $START_NAMESPACE -r -A ||
+ error "(8) Fail to start LFSCK for namespace"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(9) MDS${k} is not the expected 'completed'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_namespace |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(10) Expect 1 fixed on mds1, but got: $repaired"
+
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_namespace |
+ awk '/^lost_dirent_repaired/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(11) Expect 0 fixed on mds2, but got: $repaired"
+}
+run_test 28 "Skip the failed MDT(s) when handle orphan MDT-objects"
+
$LCTL set_param debug=-lfsck > /dev/null || true
# restore MDS/OST size
git://git.whamcloud.com / fs / lustre-release.git / blobdiff