set -e
ONLY=${ONLY:-"$*"}
+
+#Bug number for excepting test
ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
+
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
# do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
MDSSIZE=100000
OSTSIZE=100000
-# no need too much OSTs, to reduce the format/start/stop overhead
+# no need too many OSTs, to reduce the format/start/stop overhead
[ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
# build up a clean test environment.
build_test_filter
-$LCTL set_param debug=+lfsck > /dev/null || true
-
MDT_DEV="${FSNAME}-MDT0000"
OST_DEV="${FSNAME}-OST0000"
MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
echo "stop $SINGLEMDS"
stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
echo "start $SINGLEMDS"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(5) Fail to start MDS!"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 30 || {
error "(4) unexpected status"
}
+ umount_client $MOUNT
echo "stop $SINGLEMDS"
stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
echo "start $SINGLEMDS"
start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
error "(6) Fail to start MDS!"
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" 30 || {
return 0
fi
- lfsck_prep 70 70
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/lfsck || error "(1) Fail to mkdir lfsck"
+ $LFS setstripe -c 1 -i -1 $DIR/$tdir/lfsck
+ createmany -o $DIR/$tdir/lfsck/f 5000
local BASE_SPEED1=100
local RUN_TIME1=10
- $START_NAMESPACE -r -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
+ $START_LAYOUT -r -s $BASE_SPEED1 || error "(2) Fail to start LFSCK!"
sleep $RUN_TIME1
- STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ STATUS=$($SHOW_LAYOUT | awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
error "(3) Expect 'scanning-phase1', but got '$STATUS'"
- local SPEED=$($SHOW_NAMESPACE |
+ local SPEED=$($SHOW_LAYOUT |
awk '/^average_speed_phase1/ { print $2 }')
# There may be time error, normally it should be less than 2 seconds.
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
sleep $RUN_TIME2
- SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
+ SPEED=$($SHOW_LAYOUT | awk '/^average_speed_phase1/ { print $2 }')
# MIN_MARGIN = 0.8 = 8 / 10
local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
$LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
wait_update_facet $SINGLEMDS \
- "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
- awk '/^status/ { print \\\$2 }'" "completed" 30 ||
+ "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 30 ||
error "(7) Failed to get expected 'completed'"
}
run_test 9a "LFSCK speed control (1)"
return 0
fi
+ [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ { skip "Need MDS version >= 2.7.50"; return; }
+
lfsck_prep 0 0
echo "Preparing another 50 * 50 files (with error) at $(date)."
echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
#define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
do_facet ost1 $LCTL set_param fail_loc=0x160d
- createmany -o $DIR/$tdir/f 64
+
+ local count=$(precreated_ost_obj_count 0 0)
+
+ createmany -o $DIR/$tdir/f $((count + 32))
+
+ local proc_path="${FSNAME}-OST0000-osc-MDT0000"
+ local seq=$(do_facet mds1 $LCTL get_param -n \
+ osp.${proc_path}.prealloc_last_seq)
local lastid1=$(do_facet ost1 "lctl get_param -n \
- obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id" | grep $seq |
awk -F: '{ print $2 }')
umount_client $MOUNT
for ((i = 0; i < 60; i++)); do
lastid2=$(do_facet ost1 "lctl get_param -n \
- obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id" | grep $seq |
awk -F: '{ print $2 }')
[ ! -z $lastid2 ] && break;
sleep 1
echo "the on-disk LAST_ID should have been rebuilt"
wait_update_facet ost1 "$LCTL get_param -n \
- obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
+ obdfilter.${ost1_svc}.last_id | grep $seq |
awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
- $LCTL get_param -n obdfilter.${ost1_svc}.last_id
- error "(9) expect lastid1 0x100000000:$lastid1"
+ do_facet ost1 $LCTL get_param -n \
+ obdfilter.${ost1_svc}.last_id
+ error "(9) expect lastid1 $seq:$lastid1"
}
do_facet ost1 $LCTL set_param fail_loc=0
error "(7) MDS${k} is not the expected 'completed'"
done
+ start_full_debug_logging
+
echo "Start layout LFSCK on all targets by single command (-s 1)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
-s 1 -r || error "(8) Fail to start LFSCK on all devices!"
awk '/^status/ { print \\\$2 }'" "completed" 32 ||
error "(14) MDS${k} is not the expected 'completed'"
done
+
+ stop_full_debug_logging
}
run_test 12 "single command to trigger LFSCK on all devices"
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
- local count=$(precreated_ost_obj_count 0 0)
-
echo "Inject failure stub to simulate dangling referenced MDT-object"
#define OBD_FAIL_LFSCK_DANGLING 0x1610
do_facet ost1 $LCTL set_param fail_loc=0x1610
+ local count=$(precreated_ost_obj_count 0 0)
+
createmany -o $DIR/$tdir/f $((count + 31))
touch $DIR/$tdir/guard
do_facet ost1 $LCTL set_param fail_loc=0
}
run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
+test_15c() {
+ [ $MDSCOUNT -lt 2 ] &&
+ skip "We need at least 2 MDSes for this test" && return
+
+ echo "#####"
+ echo "According to current metadata migration implementation,"
+ echo "before the old MDT-object is removed, both the new MDT-object"
+ echo "and old MDT-object will reference the same LOV layout. Then if"
+ echo "the layout LFSCK finds the new MDT-object by race, it will"
+ echo "regard related OST-object(s) as multiple referenced case, and"
+ echo "will try to create new OST-object(s) for the new MDT-object."
+ echo "To avoid such trouble, the layout LFSCK needs to lock the old"
+ echo "MDT-object before confirm the multiple referenced case."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 1 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=1
+ cancel_lru_locks osc
+
+ echo "Inject failure stub on MDT1 to delay the migration"
+
+ #define OBD_FAIL_MIGRATE_DELAY 0x1803
+ do_facet mds2 $LCTL set_param fail_val=5 fail_loc=0x1803
+ echo "Migrate $DIR/$tdir/a1 from MDT1 to MDT0 with delay"
+ $LFS migrate -m 0 $DIR/$tdir/a1 &
+
+ sleep 1
+ echo "Trigger layout LFSCK to race with the migration"
+ $START_LAYOUT -A -r || error "(1) Fail to start layout LFSCK!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
+ error "(2) MDS${k} is not the expected 'completed'"
+ done
+
+ do_facet mds2 $LCTL set_param fail_loc=0 fail_val=0
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+
+ repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_multiple_referenced/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(4) Unexpectedly repaird multiple references: $repaired"
+}
+run_test 15c "LFSCK can repair unmatched MDT-object/OST-object pairs (3)"
+
test_16() {
echo "#####"
echo "If the OST-object's owner information does not match the owner"
}
run_test 17 "LFSCK can repair multiple references"
+$LCTL set_param debug=+cache > /dev/null
+
test_18a() {
echo "#####"
echo "The target MDT-object is there, but related stripe information"
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
+ start_full_debug_logging
+
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
error "(5) OST${k} Expect 'completed', but got '$cur_status'"
done
+ stop_full_debug_logging
+
local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
}
run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
+$LCTL set_param debug=-cache > /dev/null
+
test_19a() {
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
echo "#####"
echo "The parent_A references the child directory via some name entry,"
echo "but the child directory back references another parent_B via its"
- echo "".." name entry. The parent_B does not exist. Then the namesapce"
+ echo "".." name entry. The parent_B does not exist. Then the namespace"
echo "LFSCK will repair the child directory's ".." name entry."
echo "#####"
echo "The parent_A references the child directory via the name entry_B,"
echo "but the child directory back references another parent_C via its"
echo "".." name entry. The parent_C exists, but there is no the name"
- echo "entry_B under the parent_C. Then the namesapce LFSCK will repair"
+ echo "entry_B under the parent_C. Then the namespace LFSCK will repair"
echo "the child directory's ".." name entry and its linkEA."
echo "#####"
test_29c() {
echo "#####"
- echo "There are too much hard links to the object, and exceeds the
- echo object's linkEA limitation, as to NOT all the known name entries"
+ echo "There are too many hard links to the object, and exceeds the"
+ echo "object's linkEA limitation, as to NOT all the known name entries"
echo "will be recorded in the linkEA. Under such case, LFSCK should"
echo "skip the nlink verification for this object."
echo "#####"
}
run_test 31h "Repair the corrupted shard's name entry"
-$LCTL set_param debug=-lfsck > /dev/null || true
-
# restore MDS/OST size
MDSSIZE=${SAVED_MDSSIZE}
OSTSIZE=${SAVED_OSTSIZE}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff