test_2e()
{
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
check_mount_and_prep
return 0
fi
- [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.50) ]] ||
{ skip "Need MDS version >= 2.7.50"; return; }
check_mount_and_prep
return 0
fi
- [[ $server_version -ge $(version_code 2.7.50) ]] ||
+ [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.50) ]] ||
{ skip "Need MDS version >= 2.7.50"; return; }
lfsck_prep 0 0
run_test 11b "LFSCK can rebuild crashed last_id"
test_12a() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for test_12a" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
check_mount_and_prep
for k in $(seq $MDSCOUNT); do
echo "Inject failure stub to simulate bad lmm_oi"
#define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
- createmany -o $DIR/$tdir/f 32
+ createmany -o $DIR/$tdir/f 1
+ $LFS setstripe -E 1M -E -1 $DIR/$tdir/f1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f1"
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
local repaired=$($SHOW_LAYOUT |
awk '/^repaired_others/ { print $2 }')
- [ $repaired -eq 32 ] ||
+ [ $repaired -eq 2 ] ||
error "(3) Fail to repair crashed lmm_oi: $repaired"
}
run_test 13 "LFSCK can repair crashed lmm_oi"
-test_14() {
+test_14a() {
echo "#####"
echo "The OST-object referenced by the MDT-object should be there;"
echo "otherwise, the LFSCK should re-create the missing OST-object."
+ echo "without '--delay-create-ostobj' option."
echo "#####"
check_mount_and_prep
do_facet ost1 $LCTL set_param fail_loc=0x1610
local count=$(precreated_ost_obj_count 0 0)
- createmany -o $DIR/$tdir/f $((count + 31))
- touch $DIR/$tdir/guard
+ createmany -o $DIR/$tdir/f $((count + 16)) ||
+ error "(0.1) Fail to create $DIR/$tdir/fx"
+ touch $DIR/$tdir/guard0
+
+ for ((i = 0; i < 16; i++)); do
+ $LFS setstripe -E 512K -S 256K -o 0 -E 2M \
+ $DIR/$tdir/f_comp${i} ||
+ error "(0.2) Fail to create $DIR/$tdir/f_comp${i}"
+ done
+ touch $DIR/$tdir/guard1
+
do_facet ost1 $LCTL set_param fail_loc=0
start_full_debug_logging
# exhaust other pre-created dangling cases
count=$(precreated_ost_obj_count 0 0)
createmany -o $DIR/$tdir/a $count ||
- error "(0) Fail to create $count files."
+ error "(0.5) Fail to create $count files."
echo "'ls' should fail because of dangling referenced MDT-object"
ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
error "(4) Fail to repair dangling reference: $repaired"
echo "'stat' should fail because of not repair dangling by default"
- stat $DIR/$tdir/guard > /dev/null 2>&1 && error "(5) stat should fail"
+ stat $DIR/$tdir/guard0 > /dev/null 2>&1 &&
+ error "(5.1) stat should fail"
+ stat $DIR/$tdir/guard1 > /dev/null 2>&1 &&
+ error "(5.2) stat should fail"
echo "Trigger layout LFSCK to repair dangling reference"
$START_LAYOUT -r -c || error "(6) Fail to start LFSCK for layout!"
# a while until the target reparation has been done. LU-4970.
echo "'stat' should success after layout LFSCK repairing"
+ wait_update_facet client "stat $DIR/$tdir/guard0 |
+ awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ stat $DIR/$tdir/guard0
+ $SHOW_LAYOUT
+ error "(8.1) unexpected size"
+ }
+
+ wait_update_facet client "stat $DIR/$tdir/guard1 |
+ awk '/Size/ { print \\\$2 }'" "0" 32 || {
+ stat $DIR/$tdir/guard1
+ $SHOW_LAYOUT
+ error "(8.2) unexpected size"
+ }
+
+ repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_dangling/ { print $2 }')
+ [ $repaired -ge 32 ] ||
+ error "(9) Fail to repair dangling reference: $repaired"
+
+ stop_full_debug_logging
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+}
+run_test 14a "LFSCK can repair MDT-object with dangling LOV EA reference (1)"
+
+test_14b() {
+ echo "#####"
+ echo "The OST-object referenced by the MDT-object should be there;"
+ echo "otherwise, the LFSCK should re-create the missing OST-object."
+ echo "with '--delay-create-ostobj' option."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to simulate dangling referenced MDT-object"
+ #define OBD_FAIL_LFSCK_DANGLING 0x1610
+ do_facet ost1 $LCTL set_param fail_loc=0x1610
+ local count=$(precreated_ost_obj_count 0 0)
+
+ createmany -o $DIR/$tdir/f $((count + 31))
+ touch $DIR/$tdir/guard
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ start_full_debug_logging
+
+ # exhaust other pre-created dangling cases
+ count=$(precreated_ost_obj_count 0 0)
+ createmany -o $DIR/$tdir/a $count ||
+ error "(0) Fail to create $count files."
+
+ echo "'ls' should fail because of dangling referenced MDT-object"
+ ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
+
+ echo "Trigger layout LFSCK to find out dangling reference"
+ $START_LAYOUT -r -o -d || error "(2) Fail to start LFSCK for layout!"
+
+ wait_all_targets_blocked layout completed 3
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_dangling/ { print $2 }')
+ [ $repaired -ge 32 ] ||
+ error "(4) Fail to repair dangling reference: $repaired"
+
+ echo "'stat' should fail because of not repair dangling by default"
+ stat $DIR/$tdir/guard > /dev/null 2>&1 && error "(5) stat should fail"
+
+ echo "Trigger layout LFSCK to repair dangling reference"
+ $START_LAYOUT -r -o -c -d || error "(6) Fail to start LFSCK for layout!"
+
+ wait_all_targets_blocked layout completed 7
+
+ # There may be some async LFSCK updates in processing, wait for
+ # a while until the target reparation has been done. LU-4970.
+
+ echo "'stat' should success after layout LFSCK repairing"
wait_update_facet client "stat $DIR/$tdir/guard |
awk '/Size/ { print \\\$2 }'" "0" 32 || {
stat $DIR/$tdir/guard
error "(9) Fail to repair dangling reference: $repaired"
stop_full_debug_logging
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
}
-run_test 14 "LFSCK can repair MDT-object with dangling reference"
+run_test 14b "LFSCK can repair MDT-object with dangling LOV EA reference (2)"
test_15a() {
echo "#####"
echo "non-exist MDT-object."
#define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
- do_facet ost1 $LCTL set_param fail_loc=0x1611
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1611
dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ $LFS setstripe -E 1M -S 256K -c 1 -E -1 -S 512K -c $OSTCOUNT \
+ $DIR/$tdir/f1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f1"
+ # 'dd' will trigger punch RPC firstly on every OST-objects.
+ # So even though some OST-object will not be write by 'dd',
+ # as long as it is allocated (may be NOT allocated in pfl_3b)
+ # its layout information will be set also.
+ dd if=/dev/zero of=$DIR/$tdir/f1 bs=4K count=257
cancel_lru_locks osc
- do_facet ost1 $LCTL set_param fail_loc=0
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0
echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
$START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
local repaired=$($SHOW_LAYOUT |
awk '/^repaired_unmatched_pair/ { print $2 }')
- [ $repaired -eq 1 ] ||
+ [ $repaired -ge 3 ] ||
error "(3) Fail to repair unmatched pair: $repaired"
}
run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
echo "#####"
check_mount_and_prep
- $LFS setstripe -c 1 -i 0 $DIR/$tdir
- dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
+ mkdir -p $DIR/$tdir/0
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/0
+ dd if=/dev/zero of=$DIR/$tdir/0/guard bs=1M count=1
cancel_lru_locks osc
echo "Inject failure stub to make the OST-object to back point to"
echo "other MDT-object"
+ local stripes=1
+ [ $OSTCOUNT -ge 2 ] && stripes=2
+
#define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
- do_facet ost1 $LCTL set_param fail_loc=0x1612
- dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1612
+ dd if=/dev/zero of=$DIR/$tdir/0/f0 bs=1M count=1
+ $LFS setstripe -E 1M -S 256K -c $stripes -E 2M -S 512K -c 1 \
+ $DIR/$tdir/f1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f1"
+ dd if=/dev/zero of=$DIR/$tdir/f1 bs=1M count=2
cancel_lru_locks osc
- do_facet ost1 $LCTL set_param fail_loc=0
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0
echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
$START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
local repaired=$($SHOW_LAYOUT |
awk '/^repaired_unmatched_pair/ { print $2 }')
- [ $repaired -eq 1 ] ||
+ [ $repaired -eq 4 ] ||
error "(3) Fail to repair unmatched pair: $repaired"
}
run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
test_15c() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.55) ] &&
skip "Skip the test after 2.7.55 see LU-6437" && return
#define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0x1614
-
dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
+ cancel_lru_locks mdc
cancel_lru_locks osc
createmany -o $DIR/$tdir/f 1
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
-
+ $LFS setstripe -E 2M -S 256K -o 0 -E 4M -S 512K -o 0 \
+ $DIR/$tdir/f1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f1"
cancel_lru_locks mdc
cancel_lru_locks osc
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
+ echo "$DIR/$tdir/f1 and $DIR/$tdir/guard use the same OST-objects"
local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
[ $size -eq 1048576 ] ||
- error "(1) f0 (wrong) size should be 1048576, but got $size"
+ error "(1.1) f0 (wrong) size should be 1048576, but got $size"
+
+ size=$(ls -l $DIR/$tdir/f1 | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(1.2) f1 (wrong) size should be 1048576, but got $size"
echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
echo "and fix them"
local repaired=$($SHOW_LAYOUT |
awk '/^repaired_multiple_referenced/ { print $2 }')
- [ $repaired -eq 1 ] ||
+ [ $repaired -eq 2 ] ||
error "(4) Fail to repair multiple references: $repaired"
echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
[ $size -eq 1048576 ] ||
error "(6) guard size should be 1048576, but got $size"
+
+ echo "$DIR/$tdir/f1 and $DIR/$tdir/guard should use diff OST-objects"
+ dd if=/dev/zero of=$DIR/$tdir/f1 bs=1M count=2 ||
+ error "(7) Fail to write f1."
+ size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(8) guard size should be 1048576, but got $size"
}
run_test 17 "LFSCK can repair multiple references"
check_mount_and_prep
$LFS mkdir -i 0 $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
- local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ local saved_size1=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
$LFS path2fid $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a2/f2
fi
+ $LFS setstripe -E 1M -S 1M -o 0 -E -1 -S 1M $DIR/$tdir/f3 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f3"
+
+ dd if=/dev/zero of=$DIR/$tdir/f3 bs=1M count=2
+
+ local saved_size2=$(ls -il $DIR/$tdir/f3 | awk '{ print $6 }')
+
+ $LFS path2fid $DIR/$tdir/f3
+ $LFS getstripe $DIR/$tdir/f3
+
cancel_lru_locks osc
echo "Inject failure, to make the MDT-object lost its layout EA"
chown 1.1 $DIR/$tdir/a2/f2
fi
+ chown 1.1 $DIR/$tdir/f3
+
sync
sleep 2
echo "The file size should be incorrect since layout EA is lost"
local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
- [ "$cur_size" != "$saved_size" ] ||
+ [ "$cur_size" != "$saved_size1" ] ||
error "(1) Expect incorrect file1 size"
if [ $MDSCOUNT -ge 2 ]; then
cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
- [ "$cur_size" != "$saved_size" ] ||
+ [ "$cur_size" != "$saved_size1" ] ||
error "(2) Expect incorrect file2 size"
fi
+ cur_size=$(ls -il $DIR/$tdir/f3 | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size2" ] ||
+ error "(1.2) Expect incorrect file3 size"
+
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
local repaired=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
- [ $repaired -eq 1 ] ||
- error "(6.1) Expect 1 fixed on mds1, but got: $repaired"
+ [ $repaired -eq 3 ] ||
+ error "(6.1) Expect 3 fixed on mds1, but got: $repaired"
if [ $MDSCOUNT -ge 2 ]; then
repaired=$(do_facet mds2 $LCTL get_param -n \
$LFS getstripe $DIR/$tdir/a2/f2
fi
+ $LFS path2fid $DIR/$tdir/f3
+ $LFS getstripe $DIR/$tdir/f3
+
echo "The file size should be correct after layout LFSCK scanning"
cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(7) Expect file1 size $saved_size, but got $cur_size"
+ [ "$cur_size" == "$saved_size1" ] ||
+ error "(7) Expect file1 size $saved_size1, but got $cur_size"
if [ $MDSCOUNT -ge 2 ]; then
cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(8) Expect file2 size $saved_size, but got $cur_size"
+ [ "$cur_size" == "$saved_size1" ] ||
+ error "(8) Expect file2 size $saved_size1, but got $cur_size"
fi
+
+ cur_size=$(ls -il $DIR/$tdir/f3 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size2" ] ||
+ error "(9) Expect file1 size $saved_size2, but got $cur_size"
}
run_test 18a "Find out orphan OST-object and repair it (1)"
check_mount_and_prep
$LFS mkdir -i 0 $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
- local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ local saved_size1=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
echo ${fid1}
$LFS getstripe $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a2/f2
fi
+ $LFS setstripe -E 1M -S 1M -o 0 -E -1 -S 1M $DIR/$tdir/f3 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f3"
+
+ dd if=/dev/zero of=$DIR/$tdir/f3 bs=1M count=2
+
+ local saved_size2=$(ls -il $DIR/$tdir/f3 | awk '{ print $6 }')
+ local fid3=$($LFS path2fid $DIR/$tdir/f3)
+ echo ${fid3}
+ $LFS getstripe $DIR/$tdir/f3
+
cancel_lru_locks osc
echo "Inject failure, to simulate the case of missing the MDT-object"
rm -f $DIR/$tdir/a2/f2
fi
+ rm -f $DIR/$tdir/f3
+
sync
sleep 2
local repaired=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
- [ $repaired -eq 1 ] ||
- error "(4.1) Expect 1 fixed on mds1, but got: $repaired"
+ [ $repaired -eq 3 ] ||
+ error "(4.1) Expect 3 fixed on mds1, but got: $repaired"
if [ $MDSCOUNT -ge 2 ]; then
repaired=$(do_facet mds2 $LCTL get_param -n \
mv $name $DIR/$tdir/a2/f2 || error "(6) Fail to move $name"
fi
+ mv $MOUNT/.lustre/lost+found/MDT0000/${fid3}-R-0 $DIR/$tdir/f3 ||
+ error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/${fid3}-R-0"
+
$LFS path2fid $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a2/f2
fi
+ $LFS path2fid $DIR/$tdir/f3
+ $LFS getstripe $DIR/$tdir/f3
+
echo "The file size should be correct after layout LFSCK scanning"
local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(7) Expect file1 size $saved_size, but got $cur_size"
+ [ "$cur_size" == "$saved_size1" ] ||
+ error "(7) Expect file1 size $saved_size1, but got $cur_size"
if [ $MDSCOUNT -ge 2 ]; then
cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(8) Expect file2 size $saved_size, but got $cur_size"
+ [ "$cur_size" == "$saved_size1" ] ||
+ error "(8) Expect file2 size $saved_size1, but got $cur_size"
fi
+
+ cur_size=$(ls -il $DIR/$tdir/f3 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size2" ] ||
+ error "(9) Expect file1 size $saved_size2, but got $cur_size"
}
run_test 18b "Find out orphan OST-object and repair it (2)"
check_mount_and_prep
$LFS mkdir -i 0 $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
echo "Inject failure, to simulate the case of missing parent FID"
#define OBD_FAIL_LFSCK_NOPFID 0x1617
- do_facet ost1 $LCTL set_param fail_loc=0x1617
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1617
dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
$LFS getstripe $DIR/$tdir/a1/f1
if [ $MDSCOUNT -ge 2 ]; then
$LFS mkdir -i 1 $DIR/$tdir/a2
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a2
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a2
dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
$LFS getstripe $DIR/$tdir/a2/f2
fi
+ $LFS setstripe -E 1M -S 1M -o 0 -E -1 -S 1M $DIR/$tdir/f3 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f3"
+
+ dd if=/dev/zero of=$DIR/$tdir/f3 bs=1M count=2
+ $LFS getstripe $DIR/$tdir/f3
+
cancel_lru_locks osc
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0
echo "Inject failure, to simulate the case of missing the MDT-object"
#define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
rm -f $DIR/$tdir/a2/f2
fi
+ rm -f $DIR/$tdir/f3
+
sync
sleep 2
done
if [ $MDSCOUNT -ge 2 ]; then
- expected=2
+ expected=4
else
- expected=1
+ expected=3
fi
local repaired=$(do_facet mds1 $LCTL get_param -n \
test_18d() {
echo "#####"
- echo "The target MDT-object layout EA slot is occpuied by some new"
- echo "created OST-object when repair dangling reference case. Such"
- echo "conflict OST-object has never been modified. Then when found"
- echo "the orphan OST-object, LFSCK will replace it with the orphan"
- echo "OST-object."
+ echo "The target MDT-object layout EA is corrupted, but the right"
+ echo "OST-object is still alive as orphan. The layout LFSCK will"
+ echo "not create new OST-object to occupy such slot."
echo "#####"
check_mount_and_prep
mkdir $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
echo "guard" > $DIR/$tdir/a1/f1
echo "foo" > $DIR/$tdir/a1/f2
- local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+
+ echo "guard" > $DIR/$tdir/a1/f3
+ $LFS setstripe -E 1M -S 1M -o 0 -E -1 -S 1M $DIR/$tdir/a1/f4 ||
+ error "(0) Fail to create PFL $DIR/$tdir/a1/f4"
+ echo "foo" > $DIR/$tdir/a1/f4
+
+ local saved_size1=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ local saved_size2=$(ls -il $DIR/$tdir/a1/f4 | awk '{ print $6 }')
$LFS path2fid $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a1/f1
$LFS path2fid $DIR/$tdir/a1/f2
$LFS getstripe $DIR/$tdir/a1/f2
+ $LFS path2fid $DIR/$tdir/a1/f3
+ $LFS getstripe $DIR/$tdir/a1/f3
+ $LFS path2fid $DIR/$tdir/a1/f4
+ $LFS getstripe $DIR/$tdir/a1/f4
cancel_lru_locks osc
echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
echo "to reference the same OST-object (which is f1's OST-obejct)."
echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
echo "dangling reference case, but f2's old OST-object is there."
+
+ echo "The failure also makes $DIR/$tdir/a1/f3 and $DIR/$tdir/a1/f4"
+ echo "to reference the same OST-object (which is f3's OST-obejct)."
+ echo "Then drop $DIR/$tdir/a1/f3 and its OST-object, so f4 becomes"
+ echo "dangling reference case, but f4's old OST-object is there."
echo
#define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
chown 1.1 $DIR/$tdir/a1/f2
+ chown 1.1 $DIR/$tdir/a1/f4
rm -f $DIR/$tdir/a1/f1
+ rm -f $DIR/$tdir/a1/f3
sync
sleep 2
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
echo "setupall"
setupall > /dev/null
- echo "The file size should be incorrect since dangling referenced"
- local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
- [ "$cur_size" != "$saved_size" ] ||
- error "(1) Expect incorrect file2 size"
-
- #define OBD_FAIL_LFSCK_DELAY3 0x1602
- do_facet $SINGLEMDS $LCTL set_param fail_val=5 fail_loc=0x1602
-
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
- $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
-
- wait_update_facet mds1 "$LCTL get_param -n \
- mdd.$(facet_svc mds1).lfsck_layout |
- awk '/^status/ { print \\\$2 }'" "scanning-phase2" $LTIME ||
- error "(3.0) MDS1 is not the expected 'scanning-phase2'"
-
- do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+ $START_LAYOUT -r -o -c -d || error "(2) Fail to start LFSCK for layout!"
for k in $(seq $MDSCOUNT); do
# The LFSCK status query internal is 30 seconds. For the case
local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
- [ $repaired -eq 1 ] ||
- error "(5) Expect 1 orphan has been fixed, but got: $repaired"
+ [ $repaired -eq 2 ] ||
+ error "(5) Expect 2 orphans have been fixed, but got: $repaired"
+
+ repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
+ mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
+ awk '/^repaired_dangling/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(6) Expect 0 dangling has been fixed, but got: $repaired"
echo "The file size should be correct after layout LFSCK scanning"
- cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(6) Expect file2 size $saved_size, but got $cur_size"
+ local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size1" ] ||
+ error "(7) Expect file2 size $saved_size1, but got $cur_size"
+
+ cur_size=$(ls -il $DIR/$tdir/a1/f4 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size2" ] ||
+ error "(8) Expect file4 size $saved_size2, but got $cur_size"
echo "The LFSCK should find back the original data."
cat $DIR/$tdir/a1/f2
$LFS path2fid $DIR/$tdir/a1/f2
$LFS getstripe $DIR/$tdir/a1/f2
+ cat $DIR/$tdir/a1/f4
+ $LFS path2fid $DIR/$tdir/a1/f4
+ $LFS getstripe $DIR/$tdir/a1/f4
}
run_test 18d "Find out orphan OST-object and repair it (4)"
check_mount_and_prep
mkdir $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
echo "guard" > $DIR/$tdir/a1/f1
echo "foo" > $DIR/$tdir/a1/f2
- local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+
+ echo "guard" > $DIR/$tdir/a1/f3
+ $LFS setstripe -E 1M -S 1M -o 0 -E -1 -S 1M $DIR/$tdir/a1/f4 ||
+ error "(0) Fail to create PFL $DIR/$tdir/a1/f4"
+ echo "foo" > $DIR/$tdir/a1/f4
+
+ local saved_size1=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ local saved_size2=$(ls -il $DIR/$tdir/a1/f4 | awk '{ print $6 }')
+
$LFS path2fid $DIR/$tdir/a1/f1
$LFS getstripe $DIR/$tdir/a1/f1
$LFS path2fid $DIR/$tdir/a1/f2
$LFS getstripe $DIR/$tdir/a1/f2
+ $LFS path2fid $DIR/$tdir/a1/f3
+ $LFS getstripe $DIR/$tdir/a1/f3
+ $LFS path2fid $DIR/$tdir/a1/f4
+ $LFS getstripe $DIR/$tdir/a1/f4
cancel_lru_locks osc
echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
echo "to reference the same OST-object (which is f1's OST-obejct)."
echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
echo "dangling reference case, but f2's old OST-object is there."
+
+ echo "Also the failure makes $DIR/$tdir/a1/f3 and $DIR/$tdir/a1/f4"
+ echo "to reference the same OST-object (which is f3's OST-obejct)."
+ echo "Then drop $DIR/$tdir/a1/f3 and its OST-object, so f4 becomes"
+ echo "dangling reference case, but f4's old OST-object is there."
echo
#define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
chown 1.1 $DIR/$tdir/a1/f2
+ chown 1.1 $DIR/$tdir/a1/f4
rm -f $DIR/$tdir/a1/f1
+ rm -f $DIR/$tdir/a1/f3
sync
sleep 2
do_facet $SINGLEMDS $LCTL set_param fail_loc=0
echo "setupall"
setupall > /dev/null
- echo "The file size should be incorrect since dangling referenced"
- local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
- [ "$cur_size" != "$saved_size" ] ||
- error "(1) Expect incorrect file2 size"
-
#define OBD_FAIL_LFSCK_DELAY3 0x1602
do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
sync
sleep 2
- echo "Write new data to f2 to modify the new created OST-object."
- echo "dummy" >> $DIR/$tdir/a1/f2
+ echo "Write new data to f2/f4 to modify the new created OST-object."
+ echo "dummy" >> $DIR/$tdir/a1/f2 || error "write a1/f2 failed"
+ echo "dummy" >> $DIR/$tdir/a1/f4 || error "write a1/f4 failed"
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
- [ $repaired -eq 1 ] ||
- error "(6) Expect 1 orphan has been fixed, but got: $repaired"
+ [ $repaired -eq 2 ] ||
+ error "(6) Expect 2 orphans have been fixed, but got: $repaired"
echo "There should be stub file under .lustre/lost+found/MDT0000/"
[ -d $MOUNT/.lustre/lost+found/MDT0000 ] ||
error "(7) $MOUNT/.lustre/lost+found/MDT0000/ should be there"
- cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-C-*)
- [ ! -z "$cname" ] ||
- error "(8) .lustre/lost+found/MDT0000/ should not be empty"
+ local count=$(ls -l $MOUNT/.lustre/lost+found/MDT0000/*-C-* | wc -l)
+ if [ $count -ne 2 ]; then
+ ls -l $MOUNT/.lustre/lost+found/MDT0000/*-C-*
+ error "(8) Expect 2 stubs under lost+found, but got $count"
+ fi
+
+ echo "The stub file should keep the original f2 or f4 data"
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-C-* | head -n 1)
+ local cur_size=$(ls -il $cname | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size1" -a "$cur_size" != "$saved_size2" ] &&
+ error "(9) Got unexpected $cur_size"
+
+ cat $cname
+ $LFS path2fid $cname
+ $LFS getstripe $cname
- echo "The stub file should keep the original f2 data"
+ cname=$(find $MOUNT/.lustre/lost+found/MDT0000/ -name *-C-* | tail -n 1)
cur_size=$(ls -il $cname | awk '{ print $6 }')
- [ "$cur_size" == "$saved_size" ] ||
- error "(9) Expect file2 size $saved_size, but got $cur_size"
+ [ "$cur_size" != "$saved_size1" -a "$cur_size" != "$saved_size2" ] &&
+ error "(10) Got unexpected $cur_size"
cat $cname
$LFS path2fid $cname
$LFS getstripe $cname
- echo "The f2 should contains new data."
+ echo "The f2/f4 should contains new data."
cat $DIR/$tdir/a1/f2
$LFS path2fid $DIR/$tdir/a1/f2
$LFS getstripe $DIR/$tdir/a1/f2
+ cat $DIR/$tdir/a1/f4
+ $LFS path2fid $DIR/$tdir/a1/f4
+ $LFS getstripe $DIR/$tdir/a1/f4
}
run_test 18e "Find out orphan OST-object and repair it (5)"
test_18f() {
- [ $OSTCOUNT -lt 2 ] &&
- skip "The test needs at least 2 OSTs" && return
+ [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
echo "#####"
echo "The target MDT-object is lost. The LFSCK should re-create the"
check_mount_and_prep
$LFS mkdir -i 0 $DIR/$tdir/a1
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a1
dd if=/dev/zero of=$DIR/$tdir/a1/guard bs=1M count=2
dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
$LFS mkdir -i 0 $DIR/$tdir/a2
if [ $MDSCOUNT -ge 2 ]; then
$LFS mkdir -i 1 $DIR/$tdir/a3
- $LFS setstripe -c 1 -i 0 -S 1M $DIR/$tdir/a3
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir/a3
dd if=/dev/zero of=$DIR/$tdir/a3/guard bs=1M count=2
dd if=/dev/zero of=$DIR/$tdir/a3/f3 bs=1M count=2
$LFS mkdir -i 1 $DIR/$tdir/a4
}
run_test 18f "Skip the failed OST(s) when handle orphan OST-objects"
+test_18g() {
+ echo "#####"
+ echo "The target MDT-object is lost, but related OI mapping is there"
+ echo "The LFSCK should recreate the lost MDT-object without affected"
+ echo "by the stale OI mapping."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c -1 -i 0 -S 1M $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=$OSTCOUNT
+ local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
+ echo ${fid1}
+ $LFS getstripe $DIR/$tdir/a1/f1
+ cancel_lru_locks osc
+
+ echo "Inject failure to simulate lost MDT-object but keep OI mapping"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ2 0x162e
+ do_facet mds1 $LCTL set_param fail_loc=0x162e
+ rm -f $DIR/$tdir/a1/f1
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
+ error "(2) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(3) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq $OSTCOUNT ] ||
+ error "(4) Expect $OSTCOUNT fixed, but got: $repaired"
+
+ echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
+ mv $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0 $DIR/$tdir/a1/f1 ||
+ error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0"
+
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+}
+run_test 18g "Find out orphan OST-object and repair it (7)"
+
+test_18h() {
+ echo "#####"
+ echo "The PFL extent crashed. During the first cycle LFSCK scanning,"
+ echo "the layout LFSCK will keep the bad PFL file(s) there without"
+ echo "scanning its OST-object(s). Then in the second stage scanning,"
+ echo "the OST will return related OST-object(s) to the MDT as orphan."
+ echo "And then the LFSCK on the MDT can rebuild the PFL extent with"
+ echo "the 'orphan(s)' stripe information."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS setstripe -E 2M -c 1 -E -1 $DIR/$tdir/f0 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f0"
+
+ cat $LUSTRE/tests/test-framework.sh > $DIR/$tdir/f0 ||
+ error "(1.1) Fail to write $DIR/$tdir/f0"
+
+ dd if=$LUSTRE/tests/test-framework.sh of=$DIR/$tdir/f0 bs=1M seek=2 ||
+ error "(1.2) Fail to write $DIR/$tdir/f0"
+
+ cp $DIR/$tdir/f0 $DIR/$tdir/guard
+
+ echo "Inject failure stub to simulate bad PFL extent range"
+ #define OBD_FAIL_LFSCK_BAD_PFL_RANGE 0x162f
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x162f
+
+ chown 1.1 $DIR/$tdir/f0
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1 &&
+ error "(2) Write to bad PFL file should fail"
+
+ echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
+ $START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" $LTIME ||
+ error "(4.1) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(4.2) OST${k} Expect 'completed', but got '$cur_status'"
+
+ done
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 2 ] ||
+ error "(5) Fail to repair crashed PFL range: $repaired"
+
+ echo "Data in $DIR/$tdir/f0 should not be broken"
+ diff $DIR/$tdir/f0 $DIR/$tdir/guard ||
+ error "(6) Data in $DIR/$tdir/f0 is broken"
+
+ echo "Write should succeed after LFSCK repairing the bad PFL range"
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1 ||
+ error "(7) Write should succeed after LFSCK"
+}
+run_test 18h "LFSCK can repair crashed PFL extent range"
+
$LCTL set_param debug=-cache > /dev/null
test_19a() {
check_mount_and_prep
$LFS setstripe -c 1 -i 0 $DIR/$tdir
- echo "foo" > $DIR/$tdir/a0
- echo "guard" > $DIR/$tdir/a1
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 0
+
+ echo "foo1" > $DIR/$tdir/a0
+ $LFS setstripe -E 512K -S 512K -o 0 -E -1 -S 1M $DIR/$tdir/a1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/a1"
+ echo "foo2" > $DIR/$tdir/a1
+ echo "guard" > $DIR/$tdir/a2
cancel_lru_locks osc
echo "Inject failure, then client will offer wrong parent FID when read"
- do_facet ost1 $LCTL set_param -n \
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param -n \
obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
+
#define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
$LCTL set_param fail_loc=0x1619
echo "Read RPC with wrong parent FID should be denied"
- cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
+ cat $DIR/$tdir/a0 && error "(3.1) Read a0 should be denied!"
+ cat $DIR/$tdir/a1 && error "(3.2) Read a1 should be denied!"
$LCTL set_param fail_loc=0
}
run_test 19a "OST-object inconsistency self detect"
echo "Inject failure stub to make the OST-object to back point to"
echo "non-exist MDT-object"
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 0
+
#define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
- do_facet ost1 $LCTL set_param fail_loc=0x1611
- echo "foo" > $DIR/$tdir/f0
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1611
+ echo "foo1" > $DIR/$tdir/f0
+ $LFS setstripe -E 1M -S 1M -o 0 -E 4M -S 256K $DIR/$tdir/f1 ||
+ error "(0) Fail to create PFL $DIR/$tdir/f1"
+ echo "foo2" > $DIR/$tdir/f1
cancel_lru_locks osc
- do_facet ost1 $LCTL set_param fail_loc=0
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0
+ do_facet ost1 $LCTL set_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 0
echo "Nothing should be fixed since self detect and repair is disabled"
local repaired=$(do_facet ost1 $LCTL get_param -n \
obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
echo "Read RPC with right parent FID should be accepted,"
echo "and cause parent FID on OST to be fixed"
- do_facet ost1 $LCTL set_param -n \
+ do_nodes $(comma_list $(osts_nodes)) $LCTL set_param -n \
obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
- cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
+
+ cat $DIR/$tdir/f0 || error "(2.1) Read f0 should not be denied!"
+ cat $DIR/$tdir/f1 || error "(2.2) Read f1 should not be denied!"
repaired=$(do_facet ost1 $LCTL get_param -n \
obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
awk '/^repaired/ { print $2 }')
- [ $repaired -eq 1 ] ||
+ [ $repaired -eq 2 ] ||
error "(3) Expected 1 repaired, but got $repaired"
}
run_test 19b "OST-object inconsistency self repair"
-test_20() {
- [ $OSTCOUNT -lt 2 ] &&
- skip "The test needs at least 2 OSTs" && return
+PATTERN_WITH_HOLE="40000001"
+PATTERN_WITHOUT_HOLE="1"
+
+test_20a() {
+ [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
echo "#####"
echo "The target MDT-object and some of its OST-object are lost."
sleep 2
do_facet mds1 $LCTL set_param fail_loc=0 fail_val=0
- echo "Inject failure to slow down the LFSCK on OST0"
- #define OBD_FAIL_LFSCK_DELAY5 0x161b
- do_facet ost1 $LCTL set_param fail_loc=0x161b
-
echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
$START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
- sleep 3
- do_facet ost1 $LCTL set_param fail_loc=0
-
for k in $(seq $MDSCOUNT); do
# The LFSCK status query internal is 30 seconds. For the case
# of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
$LFS getstripe -v $name || error "(5.1) cannot getstripe on $name"
- local pattern=0x$($LFS getstripe -L $name)
- [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] ||
+ local pattern=$($LFS getstripe -L $name)
+ [[ "$pattern" = "$PATTERN_WITHOUT_HOLE" ]] ||
error "(5.2) NOT expect pattern flag hole, but got $pattern"
local stripes=$($LFS getstripe -c $name)
$LFS getstripe -v $name || error "(6.1) cannot getstripe on $name"
- pattern=0x$($LFS getstripe -L $name)
- [[ $((pattern & LOV_PATTERN_F_HOLE)) -ne 0 ]] ||
+ pattern=$($LFS getstripe -L $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
error "(6.2) expect pattern flag hole, but got $pattern"
stripes=$($LFS getstripe -c $name)
$LFS getstripe -v $name || error "(7.1) cannot getstripe on $name"
- pattern=0x$($LFS getstripe -L $name)
+ pattern=$($LFS getstripe -L $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(7.2) expect pattern flag hole, but got $pattern"
+
stripes=$($LFS getstripe -c $name)
size=$(stat $name | awk '/Size:/ { print $2 }')
if [ $OSTCOUNT -gt 2 ]; then
- [[ $((pattern & LOV_PATTERN_F_HOLE)) -ne 0 ]] ||
- error "(7.2.1) expect pattern flag hole, but got $pattern"
-
[ $stripes -eq 3 ] ||
error "(7.3.1) expect the stripe count is 3, but got $stripes"
touch $name || error "(7.10.1) cannot touch $name"
else
- [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] ||
- error "(7.2.2) NOT expect pattern flag hole, but got $pattern"
-
- [ $stripes -eq 1 ] ||
- error "(7.3.2) expect the stripe count is 1, but got $stripes"
+ [ $stripes -eq 2 ] ||
+ error "(7.3.2) expect the stripe count is 2, but got $stripes"
# stripe1 is dummy
[ $size -eq $((4096 * (256 + 0))) ] ||
error "(7.4.2) expect the size $((4096 * 256)), but got $size"
- cat $name > /dev/null || error "(7.5.2) cannot read $name"
+ cat $name > /dev/null &&
+ error "(7.5.2) normal read $name should fail"
+
+ failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \
+ bs=4096 2>&1 | grep "Input/output error" | wc -l)
+ [ $failures -eq 256 ] ||
+ error "(7.6.2) expect 256 IO failures, but get $failures"
+
+ bcount=$((256 * 2))
+ size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }')
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(7.7.2) expect the size $((4096 * $bcount)), got $size"
- echo "dummy" >> $name || error "(7.8.2) cannot write $name"
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 \
+ seek=256 && error "(7.8.2) write to the LOV EA hole should fail"
chown $RUNAS_ID:$RUNAS_GID $name ||
error "(7.9.2) cannot chown on $name"
$LFS getstripe -v $name || error "(8.1) cannot getstripe on $name"
- pattern=0x$($LFS getstripe -L $name)
- [[ $((pattern & LOV_PATTERN_F_HOLE)) -eq 0 ]] ||
- error "(8.2) NOT expect pattern flag hole, but got $pattern"
+ pattern=$($LFS getstripe -L $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(8.2) expect pattern flag hole, but got $pattern"
stripes=$($LFS getstripe -c $name)
- # LFSCK does not know the old f3 had 3 stripes.
- # It only tries to find as much as possible.
- # The stripe count depends on the last stripe's offset.
- [ $stripes -eq 2 ] ||
- error "(8.3) expect the stripe count is 2, but got $stripes"
+ [ $stripes -eq 3 ] ||
+ error "(8.3) expect the stripe count is 3, but got $stripes"
size=$(stat $name | awk '/Size:/ { print $2 }')
# stripe2 is lost
[ $size -eq $((4096 * (256 + 256 + 0))) ] ||
error "(8.4) expect the size $((4096 * 512)), but got $size"
- cat $name > /dev/null || error "(8.5) cannot read $name"
+ cat $name > /dev/null &&
+ error "(8.5) normal read $name should fail"
+
+ failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \
+ bs=4096 2>&1 | grep "Input/output error" | wc -l)
+ # stripe2 is dummy
+ [ $failures -eq 256 ] ||
+ error "(8.6) expect 256 IO failures, but get $failures"
+
+ bcount=$((256 * 3))
+ size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }')
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(8.7) expect the size $((4096 * $bcount)), but got $size"
- echo "dummy" >> $name || error "(8.6) cannot write $name"
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 \
+ seek=512 && error "(8.8) write to the LOV EA hole should fail"
chown $RUNAS_ID:$RUNAS_GID $name ||
- error "(8.7) cannot chown on $name"
+ error "(8.9) cannot chown on $name"
- touch $name || error "(8.8) cannot touch $name"
+ touch $name || error "(8.10) cannot touch $name"
- rm -f $name || error "(8.9) cannot unlink $name"
+ rm -f $name || error "(8.11) cannot unlink $name"
}
-run_test 20 "Handle the orphan with dummy LOV EA slot properly"
+run_test 20a "Handle the orphan with dummy LOV EA slot properly"
+
+test_20b() {
+ [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
+
+ echo "#####"
+ echo "The target MDT-object and some of its OST-object are lost."
+ echo "The LFSCK should find out the left OST-objects and re-create"
+ echo "the MDT-object under the direcotry .lustre/lost+found/MDTxxxx/"
+ echo "with the partial OST-objects (LOV EA hole)."
+
+ echo "New client can access the file with LOV EA hole via normal"
+ echo "system tools or commands without crash the system - PFL case."
+ echo "#####"
+
+ check_mount_and_prep
+
+ $LFS setstripe -E 2M -S 1M -c 2 -E -1 -S 1M -c 2 $DIR/$tdir/f0 ||
+ error "(0) Fail to create PFL file $DIR/$tdir/f0"
+ $LFS setstripe -E 2M -S 1M -c 2 -E -1 -S 1M -c 2 $DIR/$tdir/f1 ||
+ error "(1) Fail to create PFL file $DIR/$tdir/f1"
+ $LFS setstripe -E 2M -S 1M -c 2 -E -1 -S 1M -c 2 $DIR/$tdir/f2 ||
+ error "(2) Fail to create PFL file $DIR/$tdir/f2"
+
+ local bcount=$((256 * 3 + 1))
+
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=4096 count=$bcount
+ dd if=/dev/zero of=$DIR/$tdir/f1 bs=4096 count=$bcount
+ dd if=/dev/zero of=$DIR/$tdir/f2 bs=4096 count=$bcount
+
+ local fid0=$($LFS path2fid $DIR/$tdir/f0)
+ local fid1=$($LFS path2fid $DIR/$tdir/f1)
+ local fid2=$($LFS path2fid $DIR/$tdir/f2)
+
+ echo ${fid0}
+ $LFS getstripe $DIR/$tdir/f0
+ echo ${fid1}
+ $LFS getstripe $DIR/$tdir/f1
+ echo ${fid2}
+ $LFS getstripe $DIR/$tdir/f2
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Inject failure..."
+ echo "To simulate f0 lost MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/f0
+
+ echo "To simulate the case of f1 lost MDT-object and "
+ echo "the first OST-object in each PFL component"
+ #define OBD_FAIL_LFSCK_LOST_SPEOBJ 0x161a
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x161a
+ rm -f $DIR/$tdir/f1
+
+ echo "To simulate the case of f2 lost MDT-object and "
+ echo "the second OST-object in each PFL component"
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ rm -f $DIR/$tdir/f2
+
+ sync
+ sleep 2
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(4) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(5) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 8 ] ||
+ error "(6) Expect 8 fixed on mds1, but got: $repaired"
+
+ #
+ # ${fid0}-R-0 is the old f0
+ #
+ local name="$MOUNT/.lustre/lost+found/MDT0000/${fid0}-R-0"
+ echo "Check $name, which is the old f0"
+
+ $LFS getstripe -v $name || error "(7.1) cannot getstripe on $name"
+
+ local pattern=$($LFS getstripe -L -I1 $name)
+ [[ "$pattern" = "$PATTERN_WITHOUT_HOLE" ]] ||
+ error "(7.2.1) NOT expect pattern flag hole, but got $pattern"
+
+ pattern=$($LFS getstripe -L -I2 $name)
+ [[ "$pattern" = "$PATTERN_WITHOUT_HOLE" ]] ||
+ error "(7.2.2) NOT expect pattern flag hole, but got $pattern"
+
+ local stripes=$($LFS getstripe -c -I1 $name)
+ [ $stripes -eq 2 ] ||
+ error "(7.3.1) expect 2 stripes, but got $stripes"
+
+ stripes=$($LFS getstripe -c -I2 $name)
+ [ $stripes -eq 2 ] ||
+ error "(7.3.2) expect 2 stripes, but got $stripes"
+
+ local e_start=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 0 ] ||
+ error "(7.4.1) expect the COMP1 start at 0, got $e_start"
+
+ local e_end=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ $e_end -eq 2097152 ] ||
+ error "(7.4.2) expect the COMP1 end at 2097152, got $e_end"
+
+ e_start=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 2097152 ] ||
+ error "(7.5.1) expect the COMP2 start at 2097152, got $e_start"
+
+ e_end=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ "$e_end" = "EOF" ] ||
+ error "(7.5.2) expect the COMP2 end at (EOF), got $e_end"
+
+ local size=$(stat $name | awk '/Size:/ { print $2 }')
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(7.6) expect the size $((4096 * $bcount)), but got $size"
+
+ cat $name > /dev/null || error "(7.7) cannot read $name"
+
+ echo "dummy" >> $name || error "(7.8) cannot write $name"
+
+ chown $RUNAS_ID:$RUNAS_GID $name || error "(7.9) cannot chown on $name"
+
+ touch $name || error "(7.10) cannot touch $name"
+
+ rm -f $name || error "(7.11) cannot unlink $name"
+
+ #
+ # ${fid1}-R-0 contains the old f1's second stripe in each COMP
+ #
+ name="$MOUNT/.lustre/lost+found/MDT0000/${fid1}-R-0"
+ echo "Check $name, it contains f1's second OST-object in each COMP"
+
+ $LFS getstripe -v $name || error "(8.1) cannot getstripe on $name"
+
+ pattern=$($LFS getstripe -L -I1 $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(8.2.1) expect pattern flag hole, but got $pattern"
+
+ pattern=$($LFS getstripe -L -I2 $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(8.2.2) expect pattern flag hole, but got $pattern"
+
+ stripes=$($LFS getstripe -c -I1 $name)
+ [ $stripes -eq 2 ] ||
+ error "(8.3.2) expect 2 stripes, but got $stripes"
+
+ stripes=$($LFS getstripe -c -I2 $name)
+ [ $stripes -eq 2 ] ||
+ error "(8.3.2) expect 2 stripes, but got $stripes"
+
+ e_start=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 0 ] ||
+ error "(8.4.1) expect the COMP1 start at 0, got $e_start"
+
+ e_end=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ $e_end -eq 2097152 ] ||
+ error "(8.4.2) expect the COMP1 end at 2097152, got $e_end"
+
+ e_start=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 2097152 ] ||
+ error "(8.5.1) expect the COMP2 start at 2097152, got $e_start"
+
+ e_end=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ "$e_end" = "EOF" ] ||
+ error "(8.5.2) expect the COMP2 end at (EOF), got $e_end"
+
+ size=$(stat $name | awk '/Size:/ { print $2 }')
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(8.6) expect the size $((4096 * $bcount)), but got $size"
+
+ cat $name > /dev/null && error "(8.7) normal read $name should fail"
+
+ local failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \
+ bs=4096 2>&1 | grep "Input/output error" | wc -l)
+
+ # The first stripe in each COMP was lost
+ [ $failures -eq 512 ] ||
+ error "(8.8) expect 512 IO failures, but get $failures"
+
+ size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }')
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(8.9) expect the size $((4096 * $bcount)), but got $size"
+
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 &&
+ error "(8.10) write to the LOV EA hole should fail"
+
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 seek=300 ||
+ error "(8.11) write to normal stripe should NOT fail"
+
+ echo "foo" >> $name && error "(8.12) append write $name should fail"
+
+ chown $RUNAS_ID:$RUNAS_GID $name || error "(8.13) cannot chown on $name"
+
+ touch $name || error "(8.14) cannot touch $name"
+
+ rm -f $name || error "(8.15) cannot unlink $name"
+
+ #
+ # ${fid2}-R-0 contains the old f2's first stripe in each COMP
+ #
+ name="$MOUNT/.lustre/lost+found/MDT0000/${fid2}-R-0"
+ echo "Check $name, it contains f2's first stripe in each COMP"
+
+ $LFS getstripe -v $name || error "(9.1) cannot getstripe on $name"
+
+ pattern=$($LFS getstripe -L -I1 $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(9.2.1) expect pattern flag hole, but got $pattern"
+
+ pattern=$($LFS getstripe -L -I2 $name)
+ [[ "$pattern" = "$PATTERN_WITH_HOLE" ]] ||
+ error "(9.2.2) expect pattern flag hole, but got $pattern"
+
+ stripes=$($LFS getstripe -c -I1 $name)
+ [ $stripes -eq 2 ] ||
+ error "(9.3.2) expect 2 stripes, but got $stripes"
+
+ stripes=$($LFS getstripe -c -I2 $name)
+ [ $stripes -eq 2 ] ||
+ error "(9.3.2) expect 2 stripes, but got $stripes"
+
+ e_start=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 0 ] ||
+ error "(9.4.1) expect the COMP1 start at 0, got $e_start"
+
+ e_end=$($LFS getstripe -I1 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ $e_end -eq 2097152 ] ||
+ error "(9.4.2) expect the COMP1 end at 2097152, got $e_end"
+
+ e_start=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_start:/ { print $2 }')
+ [ $e_start -eq 2097152 ] ||
+ error "(9.5.1) expect the COMP2 start at 2097152, got $e_start"
+
+ e_end=$($LFS getstripe -I2 $name |
+ awk '/lcme_extent.e_end:/ { print $2 }')
+ [ "$e_end" = "EOF" ] ||
+ error "(9.5.2) expect the COMP2 end at (EOF), got $e_end"
+
+ size=$(stat $name | awk '/Size:/ { print $2 }')
+ # The second stripe in COMP was lost, so we do not know there
+ # have ever been some data before. 'stat' will regard it as
+ # no data on the lost stripe.
+ bcount=$((256 * 3))
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(9.6) expect size $((4096 * $bcount)), but got $size"
+
+ cat $name > /dev/null &&
+ error "(9.7) normal read $name should fail"
+
+ failures=$(dd if=$name of=$DIR/$tdir/dump conv=sync,noerror \
+ bs=4096 2>&1 | grep "Input/output error" | wc -l)
+ [ $failures -eq 512 ] ||
+ error "(9.8) expect 256 IO failures, but get $failures"
+
+ size=$(stat $DIR/$tdir/dump | awk '/Size:/ { print $2 }')
+ # The second stripe in COMP was lost, so we do not know there
+ # have ever been some data before. Since 'dd' skip failure,
+ # it will regard the lost stripe contains data.
+ bcount=$((256 * 4))
+ [ $size -eq $((4096 * $bcount)) ] ||
+ error "(9.9) expect the size $((4096 * $bcount)), but got $size"
+
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 \
+ seek=300 && error "(9.10) write to the LOV EA hole should fail"
+
+ dd if=/dev/zero of=$name conv=sync,notrunc bs=4096 count=1 ||
+ error "(9.11) write to normal stripe should NOT fail"
+
+ echo "foo" >> $name &&
+ error "(9.12) append write $name should fail"
+
+ chown $RUNAS_ID:$RUNAS_GID $name ||
+ error "(9.13) cannot chown on $name"
+
+ touch $name || error "(9.14) cannot touch $name"
+
+ rm -f $name || error "(7.15) cannot unlink $name"
+}
+run_test 20b "Handle the orphan with dummy LOV EA slot properly - PFL case"
test_21() {
[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.59) ]] &&
run_test 21 "run all LFSCK components by default"
test_22a() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The parent_A references the child directory via some name entry,"
run_test 22a "LFSCK can repair unmatched pairs (1)"
test_22b() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The parent_A references the child directory via the name entry_B,"
run_test 22b "LFSCK can repair unmatched pairs (2)"
test_23a() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The name entry is there, but the MDT-object for such name "
check_mount_and_prep
- [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
- # Trigger LFSCK firstly, that will generate the
- # .lustre/lost+found/MDTxxxx in advance to avoid
- # reusing the local object for the dangling name
- # entry. LU-7429
- $START_NAMESPACE -r ||
- error "(0) Fail to start LFSCK for namespace"
-
- wait_all_targets_blocked namespace completed 0.1
- }
-
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
$LFS path2fid $DIR/$tdir/d0
+ createmany -o $DIR/$tdir/d0/t 10 || error "(1.5) Fail to creatmany"
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
$LFS path2fid $DIR/$tdir/d0/f0
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
$LFS path2fid $DIR/$tdir/d0/f1
- local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
- OID=$(printf %d $OID)
+ local SEQ0=$($LFS path2fid $DIR/$tdir/d0/f0 | awk -F':' '{print $1}')
+ local SEQ1=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $1}')
- if [ $OID -eq 1 ]; then
+ if [ "$SEQ0" != "$SEQ1" ]; then
# To guarantee that the f0 and f1 are in the same FID seq
rm -f $DIR/$tdir/d0/f0 ||
error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
$LFS path2fid $DIR/$tdir/d0/f0
fi
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+ # If there is creation after the dangling injection, it may re-use
+ # the just released local object (inode) that is referenced by the
+ # dangling name entry. It will fail the dangling injection.
+ # So before deleting the target object for the dangling name entry,
+ # remove some other objects to avoid the target object being reused
+ # by some potential creations. LU-7429
+ unlinkmany $DIR/$tdir/d0/t 10 || error "(5.0) Fail to unlinkmany"
+
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
echo "'ls' should fail because of dangling name entry"
check_mount_and_prep
- [[ -d $MOUNT/.lustre/lost+found/MDT0000 ]] || {
- # Trigger LFSCK firstly, that will generate the
- # .lustre/lost+found/MDTxxxx in advance to avoid
- # reusing the local object for the dangling name
- # entry. LU-7429
- $START_NAMESPACE -r ||
- error "(0) Fail to start LFSCK for namespace"
-
- wait_all_targets_blocked namespace completed 0.1
- }
-
$LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0 on MDT0"
$LFS path2fid $DIR/$tdir/d0
+ createmany -o $DIR/$tdir/d0/t 10 || error "(1.5) Fail to creatmany"
+
echo "dummy" > $DIR/$tdir/d0/f0 || error "(2) Fail to touch on MDT0"
$LFS path2fid $DIR/$tdir/d0/f0
echo "dead" > $DIR/$tdir/d0/f1 || error "(3) Fail to touch on MDT0"
$LFS path2fid $DIR/$tdir/d0/f1
- local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
- OID=$(printf %d $OID)
+ local SEQ0=$($LFS path2fid $DIR/$tdir/d0/f0 | awk -F':' '{print $1}')
+ local SEQ1=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $1}')
- if [ $OID -eq 1 ]; then
+ if [ "$SEQ0" != "$SEQ1" ]; then
# To guarantee that the f0 and f1 are in the same FID seq
rm -f $DIR/$tdir/d0/f0 ||
error "(3.1) Fail to unlink $DIR/$tdir/d0/f0"
$LFS path2fid $DIR/$tdir/d0/f0
fi
+ local OID=$($LFS path2fid $DIR/$tdir/d0/f1 | awk -F':' '{print $2}')
+ OID=$(printf %d $OID)
+
echo "Inject failure stub on MDT0 to simulate dangling name entry"
#define OBD_FAIL_LFSCK_DANGLING3 0x1621
do_facet $SINGLEMDS $LCTL set_param fail_val=$OID fail_loc=0x1621
ln $DIR/$tdir/d0/f0 $DIR/$tdir/d0/foo || error "(4) Fail to hard link"
do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+ # If there is creation after the dangling injection, it may re-use
+ # the just released local object (inode) that is referenced by the
+ # dangling name entry. It will fail the dangling injection.
+ # So before deleting the target object for the dangling name entry,
+ # remove some other objects to avoid the target object being reused
+ # by some potential creations. LU-7429
+ unlinkmany $DIR/$tdir/d0/t 10 || error "(5.0) Fail to unlinkmany"
+
rm -f $DIR/$tdir/d0/f1 || error "(5) Fail to unlink $DIR/$tdir/d0/f1"
echo "'ls' should fail because of dangling name entry"
wait_update_facet client "stat $DIR/$tdir/d0/foo |
awk '/Size/ { print \\\$2 }'" "0" $LTIME || {
- stat $DIR/$tdir/guard
+ stat $DIR/$tdir/d0/foo
$SHOW_NAMESPACE
error "(8) unexpected size"
}
run_test 23c "LFSCK can repair dangling name entry (3)"
test_24() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "Two MDT-objects back reference the same name entry via their"
test_25() {
[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
- skip "Only support to inject failure on ldiskfs" && return
+ skip "ldiskfs only test" && return
echo "#####"
echo "The file type in the name entry does not match the file type"
run_test 26a "LFSCK can add the missing local name entry back to the namespace"
test_26b() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The remote name entry back referenced by the MDT-object is lost."
run_test 27a "LFSCK can recreate the lost local parent directory as orphan"
test_27b() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "We need at least 2 MDSes for this test" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The remote parent referenced by the MDT-object linkEA is lost."
run_test 27b "LFSCK can recreate the lost remote parent directory as orphan"
test_28() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "The target name entry is lost. The LFSCK should insert the"
count=$(stat --format=%h $DIR/$tdir/d0/foo)
[ $count -eq 2 ] || error "(8) Fail to repair nlink count: $count"
}
-run_test 29a "LFSCK can repair bad nlink count (1)"
+# Disable 29a, we only allow nlink to be updated if the known linkEA
+# entries is larger than nlink count.
+#
+#run_test 29a "LFSCK can repair bad nlink count (1)"
test_29b() {
echo "#####"
}
run_test 29b "LFSCK can repair bad nlink count (2)"
-test_29c() {
+test_29c()
+{
echo "#####"
- echo "There are too many hard links to the object, and exceeds the"
- echo "object's linkEA limitation, as to NOT all the known name entries"
- echo "will be recorded in the linkEA. Under such case, LFSCK should"
- echo "skip the nlink verification for this object."
+ echo "The namespace LFSCK will create many hard links to the target"
+ echo "file as to exceed the linkEA size limitation. Under such case"
+ echo "the linkEA will be marked as overflow that will prevent the"
+ echo "target file to be migrated. Then remove some hard links to"
+ echo "make the left hard links to be held within the linkEA size"
+ echo "limitation. But before the namespace LFSCK adding all the"
+ echo "missed linkEA entries back, the overflow mark (timestamp)"
+ echo "will not be cleared."
echo "#####"
check_mount_and_prep
- $LFS mkdir -i 0 $DIR/$tdir/d0 || error "(1) Fail to mkdir d0"
- touch $DIR/$tdir/d0/foo || error "(2) Fail to create foo"
- ln $DIR/$tdir/d0/foo $DIR/$tdir/d0/h1 ||
- error "(3) Fail to hard link to $DIR/$tdir/d0/foo"
+ mkdir -p $DIR/$tdir/guard || error "(0.1) Fail to mkdir"
+ $LFS mkdir -i $((MDSCOUNT - 1)) $DIR/$tdir/foo ||
+ error "(0.2) Fail to mkdir"
+ touch $DIR/$tdir/guard/f0 || error "(1) Fail to create"
+ local oldfid=$($LFS path2fid $DIR/$tdir/guard/f0)
+
+ # define MAX_LINKEA_SIZE 4096
+ # sizeof(link_ea_header) = 24
+ # sizeof(link_ea_entry) = 18
+ # nlink_min=$(((MAX_LINKEA_SIZE - sizeof(link_ea_header)) /
+ # (sizeof(link_ea_entry) + name_length))
+ # If the average name length is 12 bytes, then 150 hard links
+ # is totally enough to overflow the linkEA
+ echo "Create 150 hard links should succeed although the linkEA overflow"
+ createmany -l $DIR/$tdir/guard/f0 $DIR/$tdir/foo/ttttttttttt 150 ||
+ error "(2) Fail to hard link"
- echo "Inject failure stub on MDT0 to simulate the case that"
- echo "foo's hard links exceed the object's linkEA limitation."
+ cancel_lru_locks mdc
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null ||
+ error "(3.1) Migrate failure"
- #define OBD_FAIL_LFSCK_LINKEA_OVERFLOW 0x1627
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1627
- ln $DIR/$tdir/d0/foo $DIR/$tdir/d0/h2 ||
- error "(4) Fail to hard link to $DIR/$tdir/d0/foo"
+ echo "The object with linkEA overflow should NOT be migrated"
+ local newfid=$($LFS path2fid $DIR/$tdir/guard/f0)
+ [ "$newfid" == "$oldfid" ] ||
+ error "(3.2) Migrate should fail: $newfid != $oldfid"
+ fi
- cancel_lru_locks mdc
+ # Remove 100 hard links, then the linkEA should have space
+ # to hold the missed linkEA entries.
+ echo "Remove 100 hard links to save space for the missed linkEA entries"
+ unlinkmany $DIR/$tdir/foo/ttttttttttt 100 || error "(4) Fail to unlink"
- local count1=$(stat --format=%h $DIR/$tdir/d0/foo)
- [ $count1 -eq 3 ] || error "(5) Stat failure: $count1"
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null ||
+ error "(5.1) Migrate failure"
- local foofid=$($LFS path2fid $DIR/$tdir/d0/foo)
- $LFS fid2path $DIR $foofid
- local count2=$($LFS fid2path $DIR $foofid | wc -l)
- [ $count2 -eq 2 ] || error "(6) Fail to inject error: $count2"
+ # The overflow timestamp is still there, so migration will fail.
+ local newfid=$($LFS path2fid $DIR/$tdir/guard/f0)
+ [ "$newfid" == "$oldfid" ] ||
+ error "(5.2) Migrate should fail: $newfid != $oldfid"
+ fi
- echo "Trigger namespace LFSCK to repair the nlink count"
+ # sleep 3 seconds to guarantee that the overflow is recognized
+ sleep 3
+
+ echo "Trigger namespace LFSCK to clear the overflow timestamp"
$START_NAMESPACE -r -A ||
- error "(7) Fail to start LFSCK for namespace"
+ error "(6) Fail to start LFSCK for namespace"
- wait_all_targets_blocked namespace completed 8
+ wait_all_targets_blocked namespace completed 7
- do_facet $SINGLEMDS $LCTL set_param fail_loc=0
local repaired=$($SHOW_NAMESPACE |
- awk '/^nlinks_repaired/ { print $2 }')
+ awk '/^linkea_overflow_cleared/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(8) Fail to clear linkea overflow: $repaired"
+
+ repaired=$($SHOW_NAMESPACE |
+ awk '/^nlinks_repaired/ { print $2 }')
[ $repaired -eq 0 ] ||
- error "(9) Repair nlink count unexpcetedly: $repaired"
+ error "(9) Unexpected nlink repaired: $repaired"
- cancel_lru_locks mdc
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS migrate -m 1 $DIR/$tdir/guard 2>/dev/null ||
+ error "(10.1) Migrate failure"
- count1=$(stat --format=%h $DIR/$tdir/d0/foo)
- [ $count1 -eq 3 ] || error "(10) Stat failure: $count1"
+ # Migration should succeed after clear the overflow timestamp.
+ local newfid=$($LFS path2fid $DIR/$tdir/guard/f0)
+ [ "$newfid" != "$oldfid" ] ||
+ error "(10.2) Migrate should succeed"
- count2=$($LFS fid2path $DIR $foofid | wc -l)
- [ $count2 -eq 2 ] ||
- error "(11) Repaired something unexpectedly: $count2"
+ ls -l $DIR/$tdir/foo > /dev/null ||
+ error "(11) 'ls' failed after migration"
+ fi
+
+ rm -f $DIR/$tdir/guard/f0 || error "(12) Fail to unlink f0"
+ rm -rf $DIR/$tdir/foo || error "(13) Fail to rmdir foo"
}
-run_test 29c "Not verify nlink attr if hark links exceed linkEA limitation"
+run_test 29c "verify linkEA size limitation"
test_30() {
[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
- skip "Only support backend /lost+found for ldiskfs" && return
+ skip "ldiskfs only test" && return
echo "#####"
echo "The namespace LFSCK will move the orphans from backend"
run_test 30 "LFSCK can recover the orphans from backend /lost+found"
test_31a() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For the name entry under a striped directory, if the name"
run_test 31a "The LFSCK can find/repair the name entry with bad name hash (1)"
test_31b() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For the name entry under a striped directory, if the name"
run_test 31b "The LFSCK can find/repair the name entry with bad name hash (2)"
test_31c() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the master MDT-object of the striped directory"
run_test 31c "Re-generate the lost master LMV EA for striped directory"
test_31d() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the master MDT-object of the striped directory"
run_test 31d "Set broken striped directory (modified after broken) as read-only"
test_31e() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the slave MDT-object of the striped directory"
run_test 31e "Re-generate the lost slave LMV EA for striped directory (1)"
test_31f() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the slave MDT-object of the striped directory"
run_test 31f "Re-generate the lost slave LMV EA for striped directory (2)"
test_31g() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the stripe index in the slave LMV EA is"
run_test 31g "Repair the corrupted slave LMV EA"
test_31h() {
- [ $MDSCOUNT -lt 2 ] &&
- skip "The test needs at least 2 MDTs" && return
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
echo "#####"
echo "For some reason, the shard's name entry in the striped"
}
run_test 32 "stop LFSCK when some OST failed"
+test_33()
+{
+ lfsck_prep 5 5
+
+ $START_LAYOUT --dryrun -o -r ||
+ error "(1) Fail to start layout LFSCK"
+ wait_all_targets_blocked layout completed 2
+
+ local PARAMS=$($SHOW_LAYOUT | awk '/^param/ { print $2 }')
+ [ "$PARAMS" == "dryrun,all_targets,orphan" ] ||
+ error "(3) Expect 'dryrun,all_targets,orphan', got '$PARAMS'"
+
+ $START_NAMESPACE -e abort -A -r ||
+ error "(4) Fail to start namespace LFSCK"
+ wait_all_targets_blocked namespace completed 5
+
+ PARAMS=$($SHOW_NAMESPACE | awk '/^param/ { print $2 }')
+ [ "$PARAMS" == "failout,all_targets" ] ||
+ error "(6) Expect 'failout,all_targets', got '$PARAMS'"
+}
+run_test 33 "check LFSCK paramters"
+
# restore MDS/OST size
MDSSIZE=${SAVED_MDSSIZE}
OSTSIZE=${SAVED_OSTSIZE}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff