set -e
ONLY=${ONLY:-"$*"}
+
+#Bug number for excepting test
ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
+
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
require_dsh_mds || exit 0
+load_modules
+
+if ! check_versions; then
+ skip "It is NOT necessary to test lfsck under interoperation mode"
+ exit 0
+fi
+
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
+ skip "Need MDS version at least 2.3.60" && exit 0
+
LTIME=${LTIME:-120}
SAVED_MDSSIZE=${MDSSIZE}
# do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
MDSSIZE=100000
OSTSIZE=100000
-# no need too much OSTs, to reduce the format/start/stop overhead
+# no need too many OSTs, to reduce the format/start/stop overhead
[ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
# build up a clean test environment.
formatall
setupall
-[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
- skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
- exit 0
-
[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.50) ]] &&
ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2d 2e 3 22 23 24 25 26 27 28 29 30 31"
-build_test_filter
+# DNE does not support striped directory on zfs-based backend yet.
+[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 31"
-$LCTL set_param debug=+lfsck > /dev/null || true
+build_test_filter
MDT_DEV="${FSNAME}-MDT0000"
OST_DEV="${FSNAME}-OST0000"
$LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
MOUNT_OPTS_SCRUB="-o user_xattr"
MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
+MOUNT_OPTS_SKIP_LFSCK="-o user_xattr,skip_lfsck"
lfsck_prep() {
local ndirs=$1
echo "prepared $(date)."
}
+run_e2fsck_on_mdt0() {
+ [ $(facet_fstype $SINGLEMDS) != ldiskfs ] && return
+
+ stop $SINGLEMDS > /dev/null || error "(0) Fail to the stop MDT0"
+ run_e2fsck $(facet_active_host $SINGLEMDS) $(mdsdevname 1) "-n" |
+ grep "Fix? no" && {
+ run_e2fsck $(facet_active_host $SINGLEMDS) $(mdsdevname 1) "-n"
+ error "(2) Detected inconsistency on MDT0"
+ }
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+ error "(3) Fail to start MDT0"
+}
+
+wait_all_targets_blocked() {
+ local com=$1
+ local status=$2
+ local err=$3
+
+ local count=$(do_facet mds1 \
+ "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000 -w |
+ awk '/^${com}_mdts_${status}/ { print \\\$2 }'")
+ [[ $count -eq $MDSCOUNT ]] || {
+ do_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000"
+ error "($err) only $count of $MDSCOUNT MDTs are in ${status}"
+ }
+}
+
+wait_all_targets() {
+ local com=$1
+ local status=$2
+ local err=$3
+
+ wait_update_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000 |
+ awk '/^${com}_mdts_${status}/ { print \\\$2 }'" \
+ "$MDSCOUNT" $LTIME || {
+ do_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000"
+ error "($err) some MDTs are not in ${status}"
+ }
+}
+
test_0() {
lfsck_prep 3 3
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed FID-in-dirent: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
error "(5) Fail to repair the missing FID-in-LMA: $repaired"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
$START_NAMESPACE -r -A || error "(3) Fail to start LFSCK for namespace!"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(4) unexpected status"
- }
+
+ wait_all_targets_blocked namespace completed 4
local repaired=$($SHOW_NAMESPACE |
awk '/^linkea_repaired/ { print $2 }')
[ $repaired -ge 9 ] ||
error "(9) Fail to re-generate FID-in-dirent: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(10) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
[ $repaired -ge 2 ] ||
error "(9) Fail to generate FID-in-dirent for IGIF: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(10) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
echo "stop $SINGLEMDS"
stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
echo "start $SINGLEMDS"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(5) Fail to start MDS!"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 30 || {
error "(4) unexpected status"
}
+ umount_client $MOUNT
echo "stop $SINGLEMDS"
stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
echo "start $SINGLEMDS"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(6) Fail to start MDS!"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 30 || {
[ "$STATUS" == "paused" ] ||
error "(20) Expect 'paused', but got '$STATUS'"
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(20.1) Fail to stop MDS!"
+
+ echo "start $SINGLEMDS without resume LFSCK"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SKIP_LFSCK > /dev/null ||
+ error "(20.2) Fail to start MDS!"
+
+ timer=0
+ while [ $timer -lt $timeout ]; do
+ STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+ mdt.${MDT_DEV}.recovery_status |
+ awk '/^status/ { print \\\$2 }'")
+ [ "$STATUS" != "RECOVERING" ] && break;
+ sleep 1
+ timer=$((timer + 1))
+ done
+
+ [ $timer != $timeout ] ||
+ error "(20.3) recovery timeout"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "paused" ] ||
+ error "(20.4) Expect 'paused', but got '$STATUS'"
+
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1602
return 0
fi
- lfsck_prep 70 70
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/lfsck || error "(1) Fail to mkdir lfsck"
+ $LFS setstripe -c 1 -i -1 $DIR/$tdir/lfsck
+ createmany -o $DIR/$tdir/lfsck/f 5000
local BASE_SPEED1=100
local RUN_TIME1=10
- $START_NAMESPACE -r -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
+ $START_LAYOUT -r -s $BASE_SPEED1 || error "(2) Fail to start LFSCK!"
sleep $RUN_TIME1
- STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
error "(3) Expect 'scanning-phase1', but got '$STATUS'"
- local SPEED=$($SHOW_NAMESPACE |
+ local SPEED=$($SHOW_LAYOUT |
awk '/^average_speed_phase1/ { print $2 }')
# There may be time error, normally it should be less than 2 seconds.
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
sleep $RUN_TIME2
- SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
+ SPEED=$($SHOW_LAYOUT | awk '/^average_speed_phase1/ { print $2 }')
# MIN_MARGIN = 0.8 = 8 / 10
local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS \
- "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
- awk '/^status/ { print \\\$2 }'" "completed" 30 ||
+ "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 30 ||
error "(7) Failed to get expected 'completed'"
}
run_test 9a "LFSCK speed control (1)"
return 0
fi
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
lfsck_prep 0 0
echo "Preparing another 50 * 50 files (with error) at $(date)."
echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
#define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
do_facet ost1 $LCTL set_param fail_loc=0x160d
- createmany -o $DIR/$tdir/f 64
+
+ local count=$(precreated_ost_obj_count 0 0)
+
+ createmany -o $DIR/$tdir/f $((count + 32))
+
+ local proc_path="${FSNAME}-OST0000-osc-MDT0000"
+ local seq=$(do_facet mds1 $LCTL get_param -n \
+ osp.${proc_path}.prealloc_last_seq)
local lastid1=$(do_facet ost1 "lctl get_param -n \
- obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id" | grep $seq |
awk -F: '{ print $2 }')
umount_client $MOUNT
for ((i = 0; i < 60; i++)); do
lastid2=$(do_facet ost1 "lctl get_param -n \
- obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id" | grep $seq |
awk -F: '{ print $2 }')
[ ! -z $lastid2 ] && break;
sleep 1
echo "the on-disk LAST_ID should have been rebuilt"
wait_update_facet ost1 "$LCTL get_param -n \
- obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id | grep $seq |
awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
- $LCTL get_param -n obdfilter.${ost1_svc}.last_id
- error "(9) expect lastid1 0x100000000:$lastid1"
+ do_facet ost1 $LCTL get_param -n \
+ obdfilter.${ost1_svc}.last_id
+ error "(9) expect lastid1 $seq:$lastid1"
}
do_facet ost1 $LCTL set_param fail_loc=0
-s 1 -r || error "(2) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'scanning-phase1' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "scanning-phase1" ] ||
- error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
- done
+ wait_all_targets namespace scanning-phase1 3
echo "Stop namespace LFSCK on all targets by single lctl command."
do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
error "(4) Fail to stop LFSCK on all devices!"
echo "All the LFSCK targets should be in 'stopped' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "stopped" ] ||
- error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
- done
+ wait_all_targets_blocked namespace stopped 5
echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
-s 0 -r || error "(6) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'completed' status."
- for k in $(seq $MDSCOUNT); do
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 8 ||
- error "(7) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked namespace completed 7
+
+ start_full_debug_logging
echo "Start layout LFSCK on all targets by single command (-s 1)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
-s 1 -r || error "(8) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'scanning-phase1' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "scanning-phase1" ] ||
- error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
- done
+ wait_all_targets layout scanning-phase1 9
echo "Stop layout LFSCK on all targets by single lctl command."
do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
error "(10) Fail to stop LFSCK on all devices!"
echo "All the LFSCK targets should be in 'stopped' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "stopped" ] ||
- error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
- done
+ wait_all_targets_blocked layout stopped 11
for k in $(seq $OSTCOUNT); do
local STATUS=$(do_facet ost${k} $LCTL get_param -n \
-s 0 -r || error "(13) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'completed' status."
- for k in $(seq $MDSCOUNT); do
- # The LFSCK status query internal is 30 seconds. For the case
- # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
- # time to guarantee the status sync up.
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(14) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked layout completed 14
+
+ stop_full_debug_logging
}
run_test 12 "single command to trigger LFSCK on all devices"
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
- local count=$(precreated_ost_obj_count 0 0)
-
echo "Inject failure stub to simulate dangling referenced MDT-object"
#define OBD_FAIL_LFSCK_DANGLING 0x1610
do_facet ost1 $LCTL set_param fail_loc=0x1610
+ local count=$(precreated_ost_obj_count 0 0)
+
createmany -o $DIR/$tdir/f $((count + 31))
touch $DIR/$tdir/guard
do_facet ost1 $LCTL set_param fail_loc=0
}
run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
+test_15c() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.55) ] &&
+ skip "Skip the test after 2.7.55 see LU-6437" && return
+
+ echo "#####"
+ echo "According to current metadata migration implementation,"
+ echo "before the old MDT-object is removed, both the new MDT-object"
+ echo "and old MDT-object will reference the same LOV layout. Then if"
+ echo "the layout LFSCK finds the new MDT-object by race, it will"
+ echo "regard related OST-object(s) as multiple referenced case, and"
+ echo "will try to create new OST-object(s) for the new MDT-object."
+ echo "To avoid such trouble, the layout LFSCK needs to lock the old"
+ echo "MDT-object before confirm the multiple referenced case."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 1 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=1
+ cancel_lru_locks osc
+
+ echo "Inject failure stub on MDT1 to delay the migration"
+
+ #define OBD_FAIL_MIGRATE_DELAY 0x1803
+ do_facet mds2 $LCTL set_param fail_val=5 fail_loc=0x1803
+ echo "Migrate $DIR/$tdir/a1 from MDT1 to MDT0 with delay"
+ $LFS migrate -m 0 $DIR/$tdir/a1 &
+
+ sleep 1
+ echo "Trigger layout LFSCK to race with the migration"
+ $START_LAYOUT -A -r || error "(1) Fail to start layout LFSCK!"
+
+ wait_all_targets_blocked layout completed 2
+
+ do_facet mds2 $LCTL set_param fail_loc=0 fail_val=0
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+
+ repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_multiple_referenced/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(4) Unexpectedly repaird multiple references: $repaired"
+}
+run_test 15c "LFSCK can repair unmatched MDT-object/OST-object pairs (3)"
+
test_16() {
echo "#####"
echo "If the OST-object's owner information does not match the owner"
}
run_test 17 "LFSCK can repair multiple references"
+$LCTL set_param debug=+cache > /dev/null
+
test_18a() {
echo "#####"
echo "The target MDT-object is there, but related stripe information"
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
+ start_full_debug_logging
+
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
error "(5) OST${k} Expect 'completed', but got '$cur_status'"
done
+ stop_full_debug_logging
+
local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
}
run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
+$LCTL set_param debug=-cache > /dev/null
+
test_19a() {
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
echo "#####"
echo "The parent_A references the child directory via some name entry,"
echo "but the child directory back references another parent_B via its"
- echo "".." name entry. The parent_A does not exist. Then the namesapce"
+ echo "".." name entry. The parent_B does not exist. Then the namespace"
echo "LFSCK will repair the child directory's ".." name entry."
echo "#####"
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^unmatched_pairs_repaired/ { print $2 }')
echo "The parent_A references the child directory via the name entry_B,"
echo "but the child directory back references another parent_C via its"
echo "".." name entry. The parent_C exists, but there is no the name"
- echo "entry_B under the parent_B. Then the namesapce LFSCK will repair"
+ echo "entry_B under the parent_C. Then the namespace LFSCK will repair"
echo "the child directory's ".." name entry and its linkEA."
echo "#####"
echo "Inject failure stub on MDT0 to simulate bad dotdot name entry"
echo "and bad linkEA. The dummy's dotdot name entry references the"
echo "guard. The dummy's linkEA references n non-exist name entry."
- #define OBD_FAIL_LFSCK_BAD_PARENT2 0x161f
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161f
+ #define OBD_FAIL_LFSCK_BAD_PARENT 0x161e
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161e
$LFS mkdir -i 0 $DIR/$tdir/foo/dummy ||
error "(3) Fail to mkdir on MDT0"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^unmatched_pairs_repaired/ { print $2 }')
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
$START_NAMESPACE -A -r -C ||
error "(9) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(10) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 10
repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
check_mount_and_prep
+ [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
+ # Trigger LFSCK firstly, that will generate the
+ # .lustre/lost+found/MDTxxxx in advance to avoid
+ # reusing the local object for the dangling name
+ # entry. LU-7429
+ $START_NAMESPACE -r ||
+ error "(0) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 0.1
+ }
+
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ $LFS path2fid $DIR/$tdir/d0
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f1
+
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
+ if [ $OID -eq 1 ]; then
+ # To guarantee that the f0 and f1 are in the same FID seq
+ rm -f $DIR/$tdir/d0/f0 ||
+ error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
+ echo "dummy" > $DIR/$tdir/d0/f0 ||
+ error "(3.2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+ fi
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
echo "LFSCK cannot replace it."
echo "#####"
+ start_full_debug_logging
+
check_mount_and_prep
+ [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
+ # Trigger LFSCK firstly, that will generate the
+ # .lustre/lost+found/MDTxxxx in advance to avoid
+ # reusing the local object for the dangling name
+ # entry. LU-7429
+ $START_NAMESPACE -r ||
+ error "(0) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 0.1
+ }
+
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ $LFS path2fid $DIR/$tdir/d0
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f1
+
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
+ if [ $OID -eq 1 ]; then
+ # To guarantee that the f0 and f1 are in the same FID seq
+ rm -f $DIR/$tdir/d0/f0 ||
+ error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
+ echo "dummy" > $DIR/$tdir/d0/f0 ||
+ error "(3.2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+ fi
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
error "(7) Fail to start LFSCK for namespace"
wait_update_facet client "stat $DIR/$tdir/d0/foo |
- awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ awk '/Size/ { print \\\$2 }'" "0" $LTIME || {
stat $DIR/$tdir/guard
$SHOW_NAMESPACE
error "(8) unexpected size"
error "(10) unexpected status"
}
+ stop_full_debug_logging
+
local repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
[ $repaired -eq 1 ] ||
mkdir $DIR/$tdir/d0/dummy || error "(2) Fail to mkdir dummy"
$LFS path2fid $DIR/$tdir/d0/dummy
- local pfid=$($LFS path2fid $DIR/$tdir/d0/dummy)
+
+ local pfid
+ if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+ pfid=$($LFS path2fid $DIR/$tdir/d0/guard)
+ else
+ pfid=$($LFS path2fid $DIR/$tdir/d0/dummy)
+ fi
touch $DIR/$tdir/d0/guard/foo ||
error "(3) Fail to touch $DIR/$tdir/d0/guard/foo"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1622
$LFS mkdir -i 0 $DIR/$tdir/d0/dummy/foo ||
error "(4) Fail to mkdir $DIR/$tdir/d0/dummy/foo"
+ $LFS path2fid $DIR/$tdir/d0/dummy/foo
local cfid=$($LFS path2fid $DIR/$tdir/d0/dummy/foo)
rmdir $DIR/$tdir/d0/dummy/foo ||
error "(5) Fail to remove $DIR/$tdir/d0/dummy/foo name entry"
$START_NAMESPACE -A -r ||
error "(7) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(8) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 8
local repaired=$($SHOW_NAMESPACE |
awk '/^multiple_referenced_repaired/ { print $2 }')
rm -f $DIR/$tdir/d0/foo || error "(4) Fail to unlink $DIR/$tdir/d0/foo"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
- ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(5) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(5) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
rmdir $DIR/$tdir/d0/foo || error "(3) Fail to rmdir $DIR/$tdir/d0/foo"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
- ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(4) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(4) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
rm -rf $DIR/$tdir/d0 || error "(5) Fail to unlink the dir d0"
- ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(6) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && error "(6) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the lost parent"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
rmdir $DIR/$tdir/d0 || error "(4) Fail to unlink the dir d0"
- ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(5) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && error "(5) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(8) Fail to start LFSCK for namespace"
- for k in $(seq $MDSCOUNT); do
- # The LFSCK status query internal is 30 seconds. For the case
- # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
- # time to guarantee the status sync up.
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(9) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked namespace completed 9
local repaired=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_namespace |
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^nlinks_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^nlinks_repaired/ { print $2 }')
test_29c() {
echo "#####"
- echo "There are too much hard links to the object, and exceeds the
- echo object's linkEA limitation, as to NOT all the known name entries"
+ echo "There are too many hard links to the object, and exceeds the"
+ echo "object's linkEA limitation, as to NOT all the known name entries"
echo "will be recorded in the linkEA. Under such case, LFSCK should"
echo "skip the nlink verification for this object."
echo "#####"
local foofid=$($LFS path2fid $DIR/$tdir/d0/foo)
$LFS fid2path $DIR $foofid
local count2=$($LFS fid2path $DIR $foofid | wc -l)
- [ $count2 -eq 2 ] || "(6) Fail to inject error: $count2"
+ [ $count2 -eq 2 ] || error "(6) Fail to inject error: $count2"
echo "Trigger namespace LFSCK to repair the nlink count"
$START_NAMESPACE -r -A ||
error "(7) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(8) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 8
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
local repaired=$($SHOW_NAMESPACE |
$START_NAMESPACE -r -A ||
error "(14) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(15) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 15
local repaired=$($SHOW_NAMESPACE |
awk '/^local_lost_found_moved/ { print $2 }')
mount_client $MOUNT || error "(17) Fail to start client!"
- stat $DIR/$tdir/foo/f0 || "(18) f0 is not recovered"
+ stat $DIR/$tdir/foo/f0 || error "(18) f0 is not recovered"
ls -ail $MOUNT/.lustre/lost+found/
$START_NAMESPACE -r -A ||
error "(3) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(4) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 4
local repaired=$($SHOW_NAMESPACE |
awk '/^name_hash_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(3) Fail to start LFSCK for namespace"
- wait_update_facet mds2 "$LCTL get_param -n \
- mdd.$(facet_svc mds2).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(4) unexpected status"
+ wait_all_targets_blocked namespace completed 4
local repaired=$(do_facet mds2 $LCTL get_param -n \
mdd.$(facet_svc mds2).lfsck_namespace |
}
run_test 31b "The LFSCK can find/repair the name entry with bad name hash (2)"
-$LCTL set_param debug=-lfsck > /dev/null || true
+test_31c() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the master MDT-object of the striped directory"
+ echo "may lost its master LMV EA. If nobody created files under the"
+ echo "master directly after the master LMV EA lost, then the LFSCK"
+ echo "should re-generate the master LMV EA."
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "master MDT-object of the striped directory lost the LMV EA."
+
+ #define OBD_FAIL_LFSCK_LOST_MASTER_LMV 0x1629
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1629
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "Trigger namespace LFSCK to re-generate master LMV EA"
+ $START_NAMESPACE -r -A ||
+ error "(2) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 3
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^striped_dirs_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to re-generate master LMV EA: $repaired"
+
+ umount_client $MOUNT || error "(5) umount failed"
+ mount_client $MOUNT || error "(6) mount failed"
+
+ local empty=$(ls $DIR/$tdir/striped_dir/)
+ [ -z "$empty" ] || error "(7) The master LMV EA is not repaired: $empty"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(8) Fail to remove the striped directory after LFSCK"
+}
+run_test 31c "Re-generate the lost master LMV EA for striped directory"
+
+test_31d() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the master MDT-object of the striped directory"
+ echo "may lost its master LMV EA. If somebody created files under the"
+ echo "master directly after the master LMV EA lost, then the LFSCK"
+ echo "should NOT re-generate the master LMV EA, instead, it should"
+ echo "change the broken striped dirctory as read-only to prevent"
+ echo "further damage"
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "master MDT-object of the striped directory lost the LMV EA."
+
+ #define OBD_FAIL_LFSCK_LOST_MASTER_LMV 0x1629
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1629
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0
+
+ umount_client $MOUNT || error "(2) umount failed"
+ mount_client $MOUNT || error "(3) mount failed"
+
+ touch $DIR/$tdir/striped_dir/dummy ||
+ error "(4) Fail to touch under broken striped directory"
+
+ echo "Trigger namespace LFSCK to find out the inconsistency"
+ $START_NAMESPACE -r -A ||
+ error "(5) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 6
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^striped_dirs_repaired/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(7) Re-generate master LMV EA unexpected: $repaired"
+
+ stat $DIR/$tdir/striped_dir/dummy ||
+ error "(8) Fail to stat $DIR/$tdir/striped_dir/dummy"
+
+ touch $DIR/$tdir/striped_dir/foo &&
+ error "(9) The broken striped directory should be read-only"
+
+ chattr -i $DIR/$tdir/striped_dir ||
+ error "(10) Fail to chattr on the broken striped directory"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(11) Fail to remove the striped directory after LFSCK"
+}
+run_test 31d "Set broken striped directory (modified after broken) as read-only"
+
+test_31e() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the slave MDT-object of the striped directory"
+ echo "may lost its slave LMV EA. The LFSCK should re-generate the"
+ echo "slave LMV EA."
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "slave MDT-object (that resides on the same MDT as the master"
+ echo "MDT-object resides on) lost the LMV EA."
+
+ #define OBD_FAIL_LFSCK_LOST_SLAVE_LMV 0x162a
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x162a fail_val=0
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0 fail_val=0
+
+ echo "Trigger namespace LFSCK to re-generate slave LMV EA"
+ $START_NAMESPACE -r -A ||
+ error "(2) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 3
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^striped_shards_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to re-generate slave LMV EA: $repaired"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(5) Fail to remove the striped directory after LFSCK"
+}
+run_test 31e "Re-generate the lost slave LMV EA for striped directory (1)"
+
+test_31f() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the slave MDT-object of the striped directory"
+ echo "may lost its slave LMV EA. The LFSCK should re-generate the"
+ echo "slave LMV EA."
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "slave MDT-object (that resides on different MDT as the master"
+ echo "MDT-object resides on) lost the LMV EA."
+
+ #define OBD_FAIL_LFSCK_LOST_SLAVE_LMV 0x162a
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x162a fail_val=1
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0 fail_val=0
+
+ echo "Trigger namespace LFSCK to re-generate slave LMV EA"
+ $START_NAMESPACE -r -A ||
+ error "(2) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 3
+
+ local repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_namespace |
+ awk '/^striped_shards_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to re-generate slave LMV EA: $repaired"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(5) Fail to remove the striped directory after LFSCK"
+}
+run_test 31f "Re-generate the lost slave LMV EA for striped directory (2)"
+
+test_31g() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the stripe index in the slave LMV EA is"
+ echo "corrupted. The LFSCK should repair the slave LMV EA."
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "slave LMV EA on the first shard of the striped directory"
+ echo "claims the same index as the second shard claims"
+
+ #define OBD_FAIL_LFSCK_BAD_SLAVE_LMV 0x162b
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x162b fail_val=0
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0 fail_val=0
+
+ echo "Trigger namespace LFSCK to repair the slave LMV EA"
+ $START_NAMESPACE -r -A ||
+ error "(2) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 3
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^striped_shards_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to repair slave LMV EA: $repaired"
+
+ umount_client $MOUNT || error "(5) umount failed"
+ mount_client $MOUNT || error "(6) mount failed"
+
+ touch $DIR/$tdir/striped_dir/foo ||
+ error "(7) Fail to touch file after the LFSCK"
+
+ rm -f $DIR/$tdir/striped_dir/foo ||
+ error "(8) Fail to unlink file after the LFSCK"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(9) Fail to remove the striped directory after LFSCK"
+}
+run_test 31g "Repair the corrupted slave LMV EA"
+
+test_31h() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "The test needs at least 2 MDTs" && return
+
+ echo "#####"
+ echo "For some reason, the shard's name entry in the striped"
+ echo "directory may be corrupted. The LFSCK should repair the"
+ echo "bad shard's name entry."
+ echo "#####"
+
+ check_mount_and_prep
+
+ echo "Inject failure stub on MDT0 to simulate the case that the"
+ echo "first shard's name entry in the striped directory claims"
+ echo "the same index as the second shard's name entry claims."
+
+ #define OBD_FAIL_LFSCK_BAD_SLAVE_NAME 0x162c
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x162c fail_val=0
+ $LFS setdirstripe -i 0 -c $MDSCOUNT $DIR/$tdir/striped_dir ||
+ error "(1) Fail to create striped directory"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x0 fail_val=0
+
+ echo "Trigger namespace LFSCK to repair the shard's name entry"
+ $START_NAMESPACE -r -A ||
+ error "(2) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 3
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^dirent_repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to repair shard's name entry: $repaired"
+
+ umount_client $MOUNT || error "(5) umount failed"
+ mount_client $MOUNT || error "(6) mount failed"
+
+ touch $DIR/$tdir/striped_dir/foo ||
+ error "(7) Fail to touch file after the LFSCK"
+
+ rm -f $DIR/$tdir/striped_dir/foo ||
+ error "(8) Fail to unlink file after the LFSCK"
+
+ rmdir $DIR/$tdir/striped_dir ||
+ error "(9) Fail to remove the striped directory after LFSCK"
+}
+run_test 31h "Repair the corrupted shard's name entry"
+
+test_32()
+{
+ lfsck_prep 5 5
+ umount_client $MOUNT
+
+ #define OBD_FAIL_LFSCK_ASSISTANT_DIRECT 0x162d
+ do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x162d
+ $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
+
+ local STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ echo "stop ost1"
+ stop ost1 > /dev/null || error "(4) Fail to stop OST1!"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+ sleep 1
+
+ echo "stop LFSCK"
+ $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
+}
+run_test 32 "stop LFSCK when some OST failed"
# restore MDS/OST size
MDSSIZE=${SAVED_MDSSIZE}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff