set -e
ONLY=${ONLY:-"$*"}
+
+#Bug number for excepting test
ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
+
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
require_dsh_mds || exit 0
+load_modules
+
+if ! check_versions; then
+ skip "It is NOT necessary to test lfsck under interoperation mode"
+ exit 0
+fi
+
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
+ skip "Need MDS version at least 2.3.60" && exit 0
+
LTIME=${LTIME:-120}
SAVED_MDSSIZE=${MDSSIZE}
formatall
setupall
-[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
- skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
- exit 0
-
[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
$LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
MOUNT_OPTS_SCRUB="-o user_xattr"
MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
+MOUNT_OPTS_SKIP_LFSCK="-o user_xattr,skip_lfsck"
lfsck_prep() {
local ndirs=$1
echo "prepared $(date)."
}
+run_e2fsck_on_mdt0() {
+ [ $(facet_fstype $SINGLEMDS) != ldiskfs ] && return
+
+ stop $SINGLEMDS > /dev/null || error "(0) Fail to the stop MDT0"
+ run_e2fsck $(facet_active_host $SINGLEMDS) $(mdsdevname 1) "-n" |
+ grep "Fix? no" && {
+ run_e2fsck $(facet_active_host $SINGLEMDS) $(mdsdevname 1) "-n"
+ error "(2) Detected inconsistency on MDT0"
+ }
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
+ error "(3) Fail to start MDT0"
+}
+
+wait_all_targets_blocked() {
+ local com=$1
+ local status=$2
+ local err=$3
+
+ local count=$(do_facet mds1 \
+ "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000 -w |
+ awk '/^${com}_mdts_${status}/ { print \\\$2 }'")
+ [[ $count -eq $MDSCOUNT ]] || {
+ do_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000"
+ error "($err) only $count of $MDSCOUNT MDTs are in ${status}"
+ }
+}
+
+wait_all_targets() {
+ local com=$1
+ local status=$2
+ local err=$3
+
+ wait_update_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000 |
+ awk '/^${com}_mdts_${status}/ { print \\\$2 }'" \
+ "$MDSCOUNT" $LTIME || {
+ do_facet mds1 "$LCTL lfsck_query -t $com -M ${FSNAME}-MDT0000"
+ error "($err) some MDTs are not in ${status}"
+ }
+}
+
test_0() {
lfsck_prep 3 3
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed FID-in-dirent: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
error "(5) Fail to repair the missing FID-in-LMA: $repaired"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
[ $repaired -eq 1 ] ||
error "(5) Fail to repair crashed linkEA: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(6) Fail to start client!"
stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
$START_NAMESPACE -r -A || error "(3) Fail to start LFSCK for namespace!"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(4) unexpected status"
- }
+
+ wait_all_targets_blocked namespace completed 4
local repaired=$($SHOW_NAMESPACE |
awk '/^linkea_repaired/ { print $2 }')
[ $repaired -ge 9 ] ||
error "(9) Fail to re-generate FID-in-dirent: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(10) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
[ $repaired -ge 2 ] ||
error "(9) Fail to generate FID-in-dirent for IGIF: $repaired"
+ run_e2fsck_on_mdt0
+
mount_client $MOUNT || error "(10) Fail to start client!"
#define OBD_FAIL_FID_LOOKUP 0x1505
[ "$STATUS" == "paused" ] ||
error "(20) Expect 'paused', but got '$STATUS'"
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(20.1) Fail to stop MDS!"
+
+ echo "start $SINGLEMDS without resume LFSCK"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SKIP_LFSCK > /dev/null ||
+ error "(20.2) Fail to start MDS!"
+
+ timer=0
+ while [ $timer -lt $timeout ]; do
+ STATUS=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+ mdt.${MDT_DEV}.recovery_status |
+ awk '/^status/ { print \\\$2 }'")
+ [ "$STATUS" != "RECOVERING" ] && break;
+ sleep 1
+ timer=$((timer + 1))
+ done
+
+ [ $timer != $timeout ] ||
+ error "(20.3) recovery timeout"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "paused" ] ||
+ error "(20.4) Expect 'paused', but got '$STATUS'"
+
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1602
return 0
fi
- lfsck_prep 70 70
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/lfsck || error "(1) Fail to mkdir lfsck"
+ $LFS setstripe -c 1 -i -1 $DIR/$tdir/lfsck
+ createmany -o $DIR/$tdir/lfsck/f 5000
local BASE_SPEED1=100
local RUN_TIME1=10
- $START_NAMESPACE -r -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
+ $START_LAYOUT -r -s $BASE_SPEED1 || error "(2) Fail to start LFSCK!"
sleep $RUN_TIME1
- STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
error "(3) Expect 'scanning-phase1', but got '$STATUS'"
- local SPEED=$($SHOW_NAMESPACE |
+ local SPEED=$($SHOW_LAYOUT |
awk '/^average_speed_phase1/ { print $2 }')
# There may be time error, normally it should be less than 2 seconds.
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
sleep $RUN_TIME2
- SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
+ SPEED=$($SHOW_LAYOUT | awk '/^average_speed_phase1/ { print $2 }')
# MIN_MARGIN = 0.8 = 8 / 10
local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS \
- "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
- awk '/^status/ { print \\\$2 }'" "completed" 30 ||
+ "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 30 ||
error "(7) Failed to get expected 'completed'"
}
run_test 9a "LFSCK speed control (1)"
return 0
fi
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
lfsck_prep 0 0
echo "Preparing another 50 * 50 files (with error) at $(date)."
umount_client $MOUNT
stop ost1 || error "(1) Fail to stop ost1"
+ # stop MDS to forget last precreated object
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(12) Fail to start MDS!"
+
#define OBD_FAIL_OST_ENOSPC 0x215
do_facet ost1 $LCTL set_param fail_loc=0x215
}
run_test 11b "LFSCK can rebuild crashed last_id"
-test_12() {
+test_12a() {
[ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for test_12" && return
+ skip "We need at least 2 MDSes for test_12a" && return
check_mount_and_prep
for k in $(seq $MDSCOUNT); do
-s 1 -r || error "(2) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'scanning-phase1' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "scanning-phase1" ] ||
- error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
- done
+ wait_all_targets namespace scanning-phase1 3
echo "Stop namespace LFSCK on all targets by single lctl command."
do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
error "(4) Fail to stop LFSCK on all devices!"
echo "All the LFSCK targets should be in 'stopped' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "stopped" ] ||
- error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
- done
+ wait_all_targets_blocked namespace stopped 5
echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
-s 0 -r || error "(6) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'completed' status."
- for k in $(seq $MDSCOUNT); do
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 8 ||
- error "(7) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked namespace completed 7
start_full_debug_logging
-s 1 -r || error "(8) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'scanning-phase1' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "scanning-phase1" ] ||
- error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
- done
+ wait_all_targets layout scanning-phase1 9
echo "Stop layout LFSCK on all targets by single lctl command."
do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
error "(10) Fail to stop LFSCK on all devices!"
echo "All the LFSCK targets should be in 'stopped' status."
- for k in $(seq $MDSCOUNT); do
- local STATUS=$(do_facet mds${k} $LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print $2 }')
- [ "$STATUS" == "stopped" ] ||
- error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
- done
+ wait_all_targets_blocked layout stopped 11
for k in $(seq $OSTCOUNT); do
local STATUS=$(do_facet ost${k} $LCTL get_param -n \
-s 0 -r || error "(13) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'completed' status."
- for k in $(seq $MDSCOUNT); do
- # The LFSCK status query internal is 30 seconds. For the case
- # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
- # time to guarantee the status sync up.
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(14) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked layout completed 14
stop_full_debug_logging
}
-run_test 12 "single command to trigger LFSCK on all devices"
+run_test 12a "single command to trigger LFSCK on all devices"
+
+test_12b() {
+ check_mount_and_prep
+
+ echo "Start LFSCK without '-M' specified."
+ do_facet mds1 $LCTL lfsck_start -A -r ||
+ error "(0) Fail to start LFSCK without '-M'"
+
+ wait_all_targets_blocked namespace completed 1
+ wait_all_targets_blocked layout completed 2
+
+ local count=$(do_facet mds1 $LCTL dl |
+ awk '{ print $3 }' | grep mdt | wc -l)
+ if [ $count -gt 1 ]; then
+ echo
+ echo "Start layout LFSCK on the node with multipe targets,"
+ echo "but not specify '-M'/'-A' option. Should get failure."
+ echo
+ do_facet mds1 $LCTL lfsck_start -t layout -r &&
+ error "(3) Start layout LFSCK should fail" || true
+ fi
+}
+run_test 12b "auto detect Lustre device"
test_13() {
echo "#####"
[ $MDSCOUNT -lt 2 ] &&
skip "We need at least 2 MDSes for this test" && return
+ [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.55) ] &&
+ skip "Skip the test after 2.7.55 see LU-6437" && return
+
echo "#####"
echo "According to current metadata migration implementation,"
echo "before the old MDT-object is removed, both the new MDT-object"
#define OBD_FAIL_MIGRATE_DELAY 0x1803
do_facet mds2 $LCTL set_param fail_val=5 fail_loc=0x1803
echo "Migrate $DIR/$tdir/a1 from MDT1 to MDT0 with delay"
- $LFS mv -M 0 $DIR/$tdir/a1 &
+ $LFS migrate -m 0 $DIR/$tdir/a1 &
sleep 1
echo "Trigger layout LFSCK to race with the migration"
$START_LAYOUT -A -r || error "(1) Fail to start layout LFSCK!"
- for k in $(seq $MDSCOUNT); do
- # The LFSCK status query internal is 30 seconds. For the case
- # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
- # time to guarantee the status sync up.
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
- error "(2) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked layout completed 2
do_facet mds2 $LCTL set_param fail_loc=0 fail_val=0
local repaired=$($SHOW_LAYOUT |
[ "$cur_size" != "$saved_size" ] ||
error "(1) Expect incorrect file2 size"
- #define OBD_FAIL_LFSCK_DELAY3 0x1602
- do_facet $SINGLEMDS $LCTL set_param fail_val=5 fail_loc=0x1602
-
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
- wait_update_facet mds1 "$LCTL get_param -n \
- mdd.$(facet_svc mds1).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" $LTIME ||
- error "(3.0) MDS1 is not the expected 'scanning-phase2'"
-
- do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
-
for k in $(seq $MDSCOUNT); do
# The LFSCK status query internal is 30 seconds. For the case
# of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
+ start_full_debug_logging
+
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
error "(5) OST${k} Expect 'completed', but got '$cur_status'"
done
+ stop_full_debug_logging
+
local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
}
run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
+test_18g() {
+ echo "#####"
+ echo "The target MDT-object is lost, but related OI mapping is there"
+ echo "The LFSCK should recreate the lost MDT-object without affected"
+ echo "by the stale OI mapping."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c -1 -i 0 -S 1M $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=$OSTCOUNT
+ local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
+ echo ${fid1}
+ $LFS getstripe $DIR/$tdir/a1/f1
+ cancel_lru_locks osc
+
+ echo "Inject failure to simulate lost MDT-object but keep OI mapping"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ2 0x162e
+ do_facet mds1 $LCTL set_param fail_loc=0x162e
+ rm -f $DIR/$tdir/a1/f1
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
+ error "(2) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(3) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq $OSTCOUNT ] ||
+ error "(4) Expect $OSTCOUNT fixed, but got: $repaired"
+
+ echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
+ mv $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0 $DIR/$tdir/a1/f1 ||
+ error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0"
+
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+}
+run_test 18g "Find out orphan OST-object and repair it (7)"
+
$LCTL set_param debug=-cache > /dev/null
test_19a() {
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^unmatched_pairs_repaired/ { print $2 }')
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^unmatched_pairs_repaired/ { print $2 }')
$START_NAMESPACE -A -r ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
$START_NAMESPACE -A -r -C ||
error "(9) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(10) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 10
repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
check_mount_and_prep
+ [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
+ # Trigger LFSCK firstly, that will generate the
+ # .lustre/lost+found/MDTxxxx in advance to avoid
+ # reusing the local object for the dangling name
+ # entry. LU-7429
+ $START_NAMESPACE -r ||
+ error "(0) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 0.1
+ }
+
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ $LFS path2fid $DIR/$tdir/d0
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f1
+
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
+ if [ $OID -eq 1 ]; then
+ # To guarantee that the f0 and f1 are in the same FID seq
+ rm -f $DIR/$tdir/d0/f0 ||
+ error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
+ echo "dummy" > $DIR/$tdir/d0/f0 ||
+ error "(3.2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+ fi
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
echo "LFSCK cannot replace it."
echo "#####"
+ start_full_debug_logging
+
check_mount_and_prep
+ [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
+ # Trigger LFSCK firstly, that will generate the
+ # .lustre/lost+found/MDTxxxx in advance to avoid
+ # reusing the local object for the dangling name
+ # entry. LU-7429
+ $START_NAMESPACE -r ||
+ error "(0) Fail to start LFSCK for namespace"
+
+ wait_all_targets_blocked namespace completed 0.1
+ }
+
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
+ $LFS path2fid $DIR/$tdir/d0
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f1
+
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
+ if [ $OID -eq 1 ]; then
+ # To guarantee that the f0 and f1 are in the same FID seq
+ rm -f $DIR/$tdir/d0/f0 ||
+ error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
+ echo "dummy" > $DIR/$tdir/d0/f0 ||
+ error "(3.2) Fail to touch on MDT0"
+ $LFS path2fid $DIR/$tdir/d0/f0
+ fi
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1621
+ do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
error "(7) Fail to start LFSCK for namespace"
wait_update_facet client "stat $DIR/$tdir/d0/foo |
- awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ awk '/Size/ { print \\\$2 }'" "0" $LTIME || {
stat $DIR/$tdir/guard
$SHOW_NAMESPACE
error "(8) unexpected size"
error "(10) unexpected status"
}
+ stop_full_debug_logging
+
local repaired=$($SHOW_NAMESPACE |
awk '/^dangling_repaired/ { print $2 }')
[ $repaired -eq 1 ] ||
$START_NAMESPACE -A -r ||
error "(7) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(8) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 8
local repaired=$($SHOW_NAMESPACE |
awk '/^multiple_referenced_repaired/ { print $2 }')
rm -f $DIR/$tdir/d0/foo || error "(4) Fail to unlink $DIR/$tdir/d0/foo"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
- ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(5) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(5) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
rmdir $DIR/$tdir/d0/foo || error "(3) Fail to rmdir $DIR/$tdir/d0/foo"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
- ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 && "(4) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0/foo > /dev/null 2>&1 &&
+ error "(4) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
rm -rf $DIR/$tdir/d0 || error "(5) Fail to unlink the dir d0"
- ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(6) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && error "(6) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the lost parent"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
rmdir $DIR/$tdir/d0 || error "(4) Fail to unlink the dir d0"
- ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && "(5) 'ls' should fail"
+ ls -ail $DIR/$tdir/d0 > /dev/null 2>&1 && error "(5) 'ls' should fail"
echo "Trigger namespace LFSCK to repair the missing remote name entry"
$START_NAMESPACE -r -A ||
error "(6) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(7) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 7
local repaired=$($SHOW_NAMESPACE |
awk '/^lost_dirent_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(8) Fail to start LFSCK for namespace"
- for k in $(seq $MDSCOUNT); do
- # The LFSCK status query internal is 30 seconds. For the case
- # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
- # time to guarantee the status sync up.
- wait_update_facet mds${k} "$LCTL get_param -n \
- mdd.$(facet_svc mds${k}).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(9) MDS${k} is not the expected 'completed'"
- done
+ wait_all_targets_blocked namespace completed 9
local repaired=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_namespace |
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^nlinks_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^nlinks_repaired/ { print $2 }')
echo "Inject failure stub on MDT0 to simulate the case that"
echo "foo's hard links exceed the object's linkEA limitation."
- #define OBD_FAIL_LFSCK_LINKEA_OVERFLOW 0x1627
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1627
ln $DIR/$tdir/d0/foo $DIR/$tdir/d0/h2 ||
error "(4) Fail to hard link to $DIR/$tdir/d0/foo"
local foofid=$($LFS path2fid $DIR/$tdir/d0/foo)
$LFS fid2path $DIR $foofid
local count2=$($LFS fid2path $DIR $foofid | wc -l)
- [ $count2 -eq 2 ] || "(6) Fail to inject error: $count2"
+ [ $count2 -eq 2 ] || error "(6) Fail to inject error: $count2"
echo "Trigger namespace LFSCK to repair the nlink count"
$START_NAMESPACE -r -A ||
error "(7) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(8) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 8
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
local repaired=$($SHOW_NAMESPACE |
awk '/^nlinks_repaired/ { print $2 }')
[ $repaired -eq 0 ] ||
[ $count2 -eq 2 ] ||
error "(11) Repaired something unexpectedly: $count2"
}
-run_test 29c "Not verify nlink attr if hark links exceed linkEA limitation"
+# disable test_29c temporarily, it will be re-enabled in subsequent patch.
+#run_test 29c "Not verify nlink attr if hard links exceed linkEA limitation"
test_30() {
[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
$START_NAMESPACE -r -A ||
error "(14) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(15) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 15
local repaired=$($SHOW_NAMESPACE |
awk '/^local_lost_found_moved/ { print $2 }')
mount_client $MOUNT || error "(17) Fail to start client!"
- stat $DIR/$tdir/foo/f0 || "(18) f0 is not recovered"
+ stat $DIR/$tdir/foo/f0 || error "(18) f0 is not recovered"
ls -ail $MOUNT/.lustre/lost+found/
$START_NAMESPACE -r -A ||
error "(3) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(4) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 4
local repaired=$($SHOW_NAMESPACE |
awk '/^name_hash_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(3) Fail to start LFSCK for namespace"
- wait_update_facet mds2 "$LCTL get_param -n \
- mdd.$(facet_svc mds2).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(4) unexpected status"
+ wait_all_targets_blocked namespace completed 4
local repaired=$(do_facet mds2 $LCTL get_param -n \
mdd.$(facet_svc mds2).lfsck_namespace |
$START_NAMESPACE -r -A ||
error "(2) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(3) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 3
local repaired=$($SHOW_NAMESPACE |
awk '/^striped_dirs_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(5) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(6) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 6
local repaired=$($SHOW_NAMESPACE |
awk '/^striped_dirs_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(2) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(3) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 3
local repaired=$($SHOW_NAMESPACE |
awk '/^striped_shards_repaired/ { print $2 }')
check_mount_and_prep
echo "Inject failure stub on MDT0 to simulate the case that the"
- echo "slave MDT-object (that resides on differnt MDT as the master"
+ echo "slave MDT-object (that resides on different MDT as the master"
echo "MDT-object resides on) lost the LMV EA."
#define OBD_FAIL_LFSCK_LOST_SLAVE_LMV 0x162a
$START_NAMESPACE -r -A ||
error "(2) Fail to start LFSCK for namespace"
- wait_update_facet mds2 "$LCTL get_param -n \
- mdd.$(facet_svc mds2).lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(3) unexpected status"
+ wait_all_targets_blocked namespace completed 3
local repaired=$(do_facet mds2 $LCTL get_param -n \
mdd.$(facet_svc mds2).lfsck_namespace |
$START_NAMESPACE -r -A ||
error "(2) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(3) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 3
local repaired=$($SHOW_NAMESPACE |
awk '/^striped_shards_repaired/ { print $2 }')
$START_NAMESPACE -r -A ||
error "(2) Fail to start LFSCK for namespace"
- wait_update_facet $SINGLEMDS "$LCTL get_param -n \
- mdd.${MDT_DEV}.lfsck_namespace |
- awk '/^status/ { print \\\$2 }'" "completed" 32 || {
- $SHOW_NAMESPACE
- error "(3) unexpected status"
- }
+ wait_all_targets_blocked namespace completed 3
local repaired=$($SHOW_NAMESPACE |
awk '/^dirent_repaired/ { print $2 }')
}
run_test 31h "Repair the corrupted shard's name entry"
+test_32()
+{
+ lfsck_prep 5 5
+ umount_client $MOUNT
+
+ #define OBD_FAIL_LFSCK_ASSISTANT_DIRECT 0x162d
+ do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x162d
+ $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
+
+ local STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ echo "stop ost1"
+ stop ost1 > /dev/null || error "(4) Fail to stop OST1!"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+ sleep 1
+
+ echo "stop LFSCK"
+ $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
+}
+run_test 32 "stop LFSCK when some OST failed"
+
# restore MDS/OST size
MDSSIZE=${SAVED_MDSSIZE}
OSTSIZE=${SAVED_OSTSIZE}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff