. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging
-[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
+# remove the check when ZFS backend iteration is ready
+[ $(facet_fstype $SINGLEMDS) != "ldiskfs" ] &&
skip "test LFSCK only for ldiskfs" && exit 0
+[ $(facet_fstype ost1) != ldiskfs ] &&
+ skip "test LFSCK only for ldiskfs" && exit 0
+
require_dsh_mds || exit 0
MCREATE=${MCREATE:-mcreate}
OSTSIZE=100000
check_and_setup_lustre
+
+[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
+ skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
+ exit 0
+
+[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
+
+[[ $(lustre_version_code ost1) -lt $(version_code 2.5.50) ]] &&
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14"
+
build_test_filter
$LCTL set_param debug=+lfsck > /dev/null || true
MDT_DEV="${FSNAME}-MDT0000"
+OST_DEV="${FSNAME}-OST0000"
MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
START_NAMESPACE="do_facet $SINGLEMDS \
$LCTL lfsck_start -M ${MDT_DEV} -t namespace"
+START_LAYOUT="do_facet $SINGLEMDS \
+ $LCTL lfsck_start -M ${MDT_DEV} -t layout"
+START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
SHOW_NAMESPACE="do_facet $SINGLEMDS \
$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
+SHOW_LAYOUT="do_facet $SINGLEMDS \
+ $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
+SHOW_LAYOUT_ON_OST="do_facet ost1 \
+ $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
MOUNT_OPTS_SCRUB="-o user_xattr"
MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
echo "formatall"
formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
if [ ! -z $igif ]; then
#define OBD_FAIL_FID_IGIF 0x1504
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
fi
- echo "setupall"
- setupall > /dev/null
-
echo "preparing... ${nfiles} * ${ndirs} files will be created."
mkdir -p $DIR/$tdir
- cp $LUSTRE/tests/*.sh $DIR/$tdir/
- for ((i=0; i<${ndirs}; i++)); do
+ cp $LUSTRE/tests/*.sh $DIR/
+ for ((i = 0; i < ${ndirs}; i++)); do
mkdir $DIR/$tdir/d${i}
touch $DIR/$tdir/f${i}
- for ((j=0; j<${nfiles}; j++)); do
+ for ((j = 0; j < ${nfiles}; j++)); do
touch $DIR/$tdir/d${i}/f${j}
done
mkdir $DIR/$tdir/e${i}
awk '/^updated_phase1/ { print $2 }')
[ $repaired -eq 0 ] ||
error "(10) Expect nothing to be repaired, but got: $repaired"
+
+ local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
+ $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
+ sleep 3
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(12) Expect 'completed', but got '$STATUS'"
+
+ local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
+ [ $((scanned1 + 1)) -eq $scanned2 ] ||
+ error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
+
+ echo "stopall, should NOT crash LU-3649"
+ stopall > /dev/null
}
run_test 0 "Control LFSCK manually"
}
run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
+test_2a() {
+ lfsck_prep 1 1
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
+ touch $DIR/$tdir/dummy
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ umount_client $MOUNT
+ $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
+
+ sleep 3
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(4) Expect 'completed', but got '$STATUS'"
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^updated_phase1/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair crashed linkEA: $repaired"
+
+ mount_client $MOUNT || error "(6) Fail to start client!"
+
+ stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
+ error "(7) Fail to stat $DIR/$tdir/dummy"
+
+ local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
+ error "(8) Fail to repair linkEA: $dummyfid $dummyname"
+}
+run_test 2a "LFSCK can find out and repair crashed linkEA entry"
+
+test_2b()
+{
+ lfsck_prep 1 1
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ touch $DIR/$tdir/dummy
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ umount_client $MOUNT
+ $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
+
+ sleep 3
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(4) Expect 'completed', but got '$STATUS'"
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^updated_phase2/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair crashed linkEA: $repaired"
+
+ mount_client $MOUNT || error "(6) Fail to start client!"
+
+ stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
+ error "(7) Fail to stat $DIR/$tdir/dummy"
+
+ local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
+ error "(8) Fail to repair linkEA: $dummyfid $dummyname"
+}
+run_test 2b "LFSCK can find out and remove invalid linkEA entry"
+
+test_2c()
+{
+ lfsck_prep 1 1
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
+ touch $DIR/$tdir/dummy
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ umount_client $MOUNT
+ $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
+
+ sleep 3
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(4) Expect 'completed', but got '$STATUS'"
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^updated_phase2/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Fail to repair crashed linkEA: $repaired"
+
+ mount_client $MOUNT || error "(6) Fail to start client!"
+
+ stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
+ error "(7) Fail to stat $DIR/$tdir/dummy"
+
+ local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
+ local dummyname=$($LFS fid2path $DIR $dummyfid)
+ [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
+ error "(8) Fail to repair linkEA: $dummyfid $dummyname"
+}
+run_test 2c "LFSCK can find out and remove repeated linkEA entry"
+
test_4()
{
lfsck_prep 3 3
- mds_backup_restore || error "(1) Fail to backup/restore!"
+ mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
echo "start $SINGLEMDS with disabling OI scrub"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
error "(2) Fail to start MDS!"
test_5()
{
lfsck_prep 1 1 1
- mds_backup_restore 1 || error "(1) Fail to backup/restore!"
+ mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
echo "start $SINGLEMDS with disabling OI scrub"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
error "(2) Fail to start MDS!"
local POSITION1=$($SHOW_NAMESPACE |
awk '/^latest_start_position/ { print $4 }')
if [ $POSITION0 -gt $POSITION1 ]; then
- [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] ||
+ [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
error "(7) Expect larger than: $POSITION0, but got $POSITION1"
fi
}
run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
+test_7b()
+{
+ lfsck_prep 2 2
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ for ((i = 0; i < 20; i++)); do
+ touch $DIR/$tdir/dummy${i}
+ done
+
+ #define OBD_FAIL_LFSCK_DELAY3 0x1602
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
+ $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
+
+ sleep 3
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase2" ] ||
+ error "(4) Expect 'scanning-phase2', but got '$STATUS'"
+
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
+
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(6) Fail to start MDS!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase2" ] ||
+ error "(7) Expect 'scanning-phase2', but got '$STATUS'"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(8) Expect 'completed', but got '$STATUS'"
+}
+run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
+
+test_8()
+{
+ lfsck_prep 20 20
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(2) Expect 'init', but got '$STATUS'"
+
+ mount_client $MOUNT || error "(3) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
+ mkdir $DIR/$tdir/crashed
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ for ((i = 0; i < 5; i++)); do
+ touch $DIR/$tdir/dummy${i}
+ done
+
+ #define OBD_FAIL_LFSCK_DELAY2 0x1601
+ do_facet $SINGLEMDS $LCTL set_param fail_val=2
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
+ $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(5) Expect 'scanning-phase1', but got '$STATUS'"
+
+ $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(7) Expect 'stopped', but got '$STATUS'"
+
+ $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(9) Expect 'scanning-phase1', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_FATAL2 0x1609
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "failed" ] ||
+ error "(10) Expect 'failed', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_DELAY1 0x1600
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
+ $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(12) Expect 'scanning-phase1', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_CRASH 0x160a
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
+ sleep 5
+
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
+
+ #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
+
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(14) Fail to start MDS!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "crashed" ] ||
+ error "(15) Expect 'crashed', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_DELAY2 0x1601
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
+ $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(17) Expect 'scanning-phase1', but got '$STATUS'"
+
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
+
+ #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
+
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(19) Fail to start MDS!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "paused" ] ||
+ error "(20) Expect 'paused', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_DELAY3 0x1602
+ do_facet $SINGLEMDS $LCTL set_param fail_val=2
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
+
+ $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
+ sleep 2
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase2" ] ||
+ error "(22) Expect 'scanning-phase2', but got '$STATUS'"
+
+ local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
+ [ "$FLAGS" == "scanned-once,inconsistent" ] ||
+ error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 2
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(24) Expect 'completed', but got '$STATUS'"
+
+ FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
+ [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
+
+}
+run_test 8 "LFSCK state machine"
+
test_9a() {
if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
skip "Testing on UP system, the speed may be inaccurate."
[ "$STATUS" == "init" ] ||
error "(2) Expect 'init', but got '$STATUS'"
- $START_NAMESPACE -s 100 || error "(3) Fail to start LFSCK!"
+ local BASE_SPEED1=100
+ local RUN_TIME1=10
+ $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
- sleep 10
+ sleep $RUN_TIME1
STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
error "(3) Expect 'scanning-phase1', but got '$STATUS'"
local SPEED=$($SHOW_NAMESPACE |
awk '/^average_speed_phase1/ { print $2 }')
- # (100 * (10 + 1)) / 10 = 110
- [ $SPEED -lt 120 ] ||
- error "(4) Unexpected speed $SPEED, should not more than 120"
+
+ # There may be time error, normally it should be less than 2 seconds.
+ # We allow another 20% schedule error.
+ local TIME_DIFF=2
+ # MAX_MARGIN = 1.2 = 12 / 10
+ local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
+ RUN_TIME1 * 12 / 10))
+ [ $SPEED -lt $MAX_SPEED ] ||
+ error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
# adjust speed limit
+ local BASE_SPEED2=300
+ local RUN_TIME2=10
do_facet $SINGLEMDS \
- $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
- sleep 10
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
+ sleep $RUN_TIME2
SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
- # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180
- [ $SPEED -lt 170 ] &&
- error "(5) Unexpected speed $SPEED, should not less than 170"
-
- # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220
- [ $SPEED -lt 230 ] ||
- error "(6) Unexpected speed $SPEED, should not more than 230"
+ # MIN_MARGIN = 0.8 = 8 / 10
+ local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
+ BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
+ (RUN_TIME1 + RUN_TIME2) * 8 / 10))
+ [ $SPEED -gt $MIN_SPEED ] ||
+ error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
+
+ # MAX_MARGIN = 1.2 = 12 / 10
+ MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
+ BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
+ (RUN_TIME1 + RUN_TIME2) * 12 / 10))
+ [ $SPEED -lt $MAX_SPEED ] ||
+ error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
do_facet $SINGLEMDS \
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
}
run_test 9a "LFSCK speed control (1)"
+test_9b() {
+ if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
+ skip "Testing on UP system, the speed may be inaccurate."
+ return 0
+ fi
+
+ lfsck_prep 0 0
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ echo "Another preparing... 50 * 50 files (with error) will be created."
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ for ((i = 0; i < 50; i++)); do
+ mkdir -p $DIR/$tdir/d${i}
+ touch $DIR/$tdir/f${i}
+ for ((j = 0; j < 50; j++)); do
+ touch $DIR/$tdir/d${i}/f${j}
+ done
+ done
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(3) Expect 'init', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
+ $START_NAMESPACE || error "(4) Fail to start LFSCK!"
+
+ sleep 10
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(5) Expect 'stopped', but got '$STATUS'"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ local BASE_SPEED1=50
+ local RUN_TIME1=10
+ $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
+
+ sleep $RUN_TIME1
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase2" ] ||
+ error "(7) Expect 'scanning-phase2', but got '$STATUS'"
+
+ local SPEED=$($SHOW_NAMESPACE |
+ awk '/^average_speed_phase2/ { print $2 }')
+ # There may be time error, normally it should be less than 2 seconds.
+ # We allow another 20% schedule error.
+ local TIME_DIFF=2
+ # MAX_MARGIN = 1.2 = 12 / 10
+ local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
+ RUN_TIME1 * 12 / 10))
+ [ $SPEED -lt $MAX_SPEED ] ||
+ error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
+
+ # adjust speed limit
+ local BASE_SPEED2=150
+ local RUN_TIME2=10
+ do_facet $SINGLEMDS \
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
+ sleep $RUN_TIME2
+
+ SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
+ # MIN_MARGIN = 0.8 = 8 / 10
+ local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
+ BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
+ (RUN_TIME1 + RUN_TIME2) * 8 / 10))
+ [ $SPEED -gt $MIN_SPEED ] ||
+ error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
+
+ # MAX_MARGIN = 1.2 = 12 / 10
+ MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
+ BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
+ (RUN_TIME1 + RUN_TIME2) * 12 / 10))
+ [ $SPEED -lt $MAX_SPEED ] ||
+ error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
+
+ do_facet $SINGLEMDS \
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ sleep 5
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(11) Expect 'completed', but got '$STATUS'"
+}
+run_test 9b "LFSCK speed control (2)"
+
+test_10()
+{
+ lfsck_prep 1 1
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ mount_client $MOUNT || error "(2) Fail to start client!"
+
+ #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
+ for ((i = 0; i < 1000; i = $((i+2)))); do
+ mkdir -p $DIR/$tdir/d${i}
+ touch $DIR/$tdir/f${i}
+ for ((j = 0; j < 5; j++)); do
+ touch $DIR/$tdir/d${i}/f${j}
+ done
+ done
+
+ #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
+ for ((i = 1; i < 1000; i = $((i+2)))); do
+ mkdir -p $DIR/$tdir/d${i}
+ touch $DIR/$tdir/f${i}
+ for ((j = 0; j < 5; j++)); do
+ touch $DIR/$tdir/d${i}/f${j}
+ done
+ done
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
+
+ umount_client $MOUNT
+ mount_client $MOUNT || error "(3) Fail to start client!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(4) Expect 'init', but got '$STATUS'"
+
+ $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
+
+ sleep 10
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(6) Expect 'scanning-phase1', but got '$STATUS'"
+
+ ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
+
+ touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
+
+ mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
+
+ unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
+
+ rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
+
+ mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
+
+ ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
+
+ ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
+ error "(14) Fail to softlink!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(15) Expect 'scanning-phase1', but got '$STATUS'"
+
+ do_facet $SINGLEMDS \
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ umount_client $MOUNT
+ sleep 10
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(16) Expect 'completed', but got '$STATUS'"
+}
+run_test 10 "System is available during LFSCK scanning"
+
+# remove LAST_ID
+ost_remove_lastid() {
+ local ost=$1
+ local idx=$2
+ local rcmd="do_facet ost${ost}"
+
+ echo "remove LAST_ID on ost${ost}: idx=${idx}"
+
+ # step 1: local mount
+ mount_fstype ost${ost} || return 1
+ # step 2: remove the specified LAST_ID
+ ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/LAST_ID
+ # step 3: umount
+ unmount_fstype ost${ost} || return 2
+}
+
+test_11a() {
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $SETSTRIPE -c 1 -i 0 $DIR/$tdir
+ createmany -o $DIR/$tdir/f 64
+
+ echo "stopall"
+ stopall > /dev/null
+
+ ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
+
+ echo "start ost1"
+ start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
+ error "(2) Fail to start ost1"
+
+ local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(3) Expect 'init', but got '$STATUS'"
+
+ #define OBD_FAIL_LFSCK_DELAY4 0x160e
+ do_facet ost1 $LCTL set_param fail_val=3
+ do_facet ost1 $LCTL set_param fail_loc=0x160e
+
+ echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
+ $START_LAYOUT_ON_OST || error "(4) Fail to start LFSCK on OST!"
+
+ wait_update_facet ost1 "$LCTL get_param -n \
+ obdfilter.${OST_DEV}.lfsck_layout |
+ awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
+ $SHOW_LAYOUT_ON_OST
+ return 5
+ }
+
+ do_facet ost1 $LCTL set_param fail_val=0
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ wait_update_facet ost1 "$LCTL get_param -n \
+ obdfilter.${OST_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || {
+ $SHOW_LAYOUT_ON_OST
+ return 6
+ }
+
+ echo "the LAST_ID(s) should have been rebuilt"
+ FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
+ [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
+}
+run_test 11a "LFSCK can rebuild lost last_id"
+
+test_11b() {
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $SETSTRIPE -c 1 -i 0 $DIR/$tdir
+
+ echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
+ #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
+ do_facet ost1 $LCTL set_param fail_loc=0x160d
+ createmany -o $DIR/$tdir/f 64
+ local lastid1=$(do_facet ost1 "lctl get_param -n \
+ obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ awk -F: '{ print $2 }')
+
+ umount_client $MOUNT
+ echo "stop ost1"
+ stop ost1 || error "(1) Fail to stop ost1"
+
+ #define OBD_FAIL_OST_ENOSPC 0x215
+ do_facet ost1 $LCTL set_param fail_loc=0x215
+
+ echo "start ost1"
+ start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
+ error "(2) Fail to start ost1"
+
+ local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(3) Expect 'init', but got '$STATUS'"
+
+ for ((i = 0; i < 60; i++)); do
+ lastid2=$(do_facet ost1 "lctl get_param -n \
+ obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ awk -F: '{ print $2 }')
+ [ ! -z $lastid2 ] && break;
+ sleep 1
+ done
+
+ echo "the on-disk LAST_ID should be smaller than the expected one"
+ [ $lastid1 -gt $lastid2 ] ||
+ error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
+
+ echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
+ $START_LAYOUT_ON_OST || error "(5) Fail to start LFSCK on OST!"
+
+ wait_update_facet ost1 "$LCTL get_param -n \
+ obdfilter.${OST_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || {
+ $SHOW_LAYOUT_ON_OST
+ return 6
+ }
+
+ echo "stop ost1"
+ stop ost1 || error "(7) Fail to stop ost1"
+
+ echo "start ost1"
+ start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
+ error "(8) Fail to start ost1"
+
+ echo "the on-disk LAST_ID should have been rebuilt"
+ wait_update_facet ost1 "$LCTL get_param -n \
+ obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
+ awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
+ $LCTL get_param -n obdfilter.${ost1_svc}.last_id
+ error "(9) expect lastid1 0x100000000:$lastid1"
+ }
+
+ do_facet ost1 $LCTL set_param fail_loc=0
+}
+run_test 11b "LFSCK can rebuild crashed last_id"
+
+test_12() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for test_12" && exit 0
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "All the LFSCK targets should be in 'init' status."
+ for k in $(seq $MDSCOUNT); do
+ local STATUS=$(do_facet mds${k} $LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(1) MDS${k} Expect 'init', but got '$STATUS'"
+
+ $LFS mkdir -i $((k - 1)) $DIR/${k}
+ createmany -o $DIR/${k}/f 100
+ done
+
+ echo "Trigger LFSCK on all targets by single command (limited speed)."
+ do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
+ -s 10 || error "(2) Fail to start LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'scanning-phase1' status."
+ for k in $(seq $MDSCOUNT); do
+ local STATUS=$(do_facet mds${k} $LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
+ done
+
+ echo "Stop layout LFSCK on all targets by single lctl command."
+ do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
+ error "(4) Fail to stop LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'stopped' status."
+ for k in $(seq $MDSCOUNT); do
+ local STATUS=$(do_facet mds${k} $LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local STATUS=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(6) OST${k} Expect 'stopped', but got '$STATUS'"
+ done
+
+ echo "Re-trigger LFSCK on all targets by single command (full speed)."
+ do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
+ -s 0 || error "(7) Fail to start LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'completed' status."
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(8) MDS${k} is not the expected 'completed'"
+ done
+}
+run_test 12 "single command to trigger LFSCK on all devices"
+
+test_13() {
+ echo "#####"
+ echo "The lmm_oi in layout EA should be consistent with the MDT-object"
+ echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
+ echo "MDT-object FID."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+
+ echo "Inject failure stub to simulate bad lmm_oi"
+ #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
+ createmany -o $DIR/$tdir/f 32
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
+ $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_others/ { print $2 }')
+ [ $repaired -eq 32 ] ||
+ error "(3) Fail to repair crashed lmm_oi: $repaired"
+}
+run_test 13 "LFSCK can repair crashed lmm_oi"
+
+test_14() {
+ echo "#####"
+ echo "The OST-object referenced by the MDT-object should be there;"
+ echo "otherwise, the LFSCK should re-create the missed OST-object."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to simulate dangling referenced MDT-object"
+ #define OBD_FAIL_LFSCK_DANGLING 0x1610
+ do_facet ost1 $LCTL set_param fail_loc=0x1610
+ createmany -o $DIR/$tdir/f 64
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "'ls' should fail because of dangling referenced MDT-object"
+ ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
+
+ echo "Trigger layout LFSCK to find out dangling reference and fix them"
+ $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 6 || return 3
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_dangling/ { print $2 }')
+ [ $repaired -eq 32 ] ||
+ error "(4) Fail to repair dangling reference: $repaired"
+
+ echo "'ls' should success after layout LFSCK repairing"
+ ls -ail $DIR/$tdir > /dev/null || error "(5) ls should success."
+}
+run_test 14 "LFSCK can repair MDT-object with dangling reference"
+
$LCTL set_param debug=-lfsck > /dev/null || true
# restore MDS/OST size
git://git.whamcloud.com / fs / lustre-release.git / blobdiff