+test_15a() {
+ echo "#####"
+ echo "If the OST-object referenced by the MDT-object back points"
+ echo "to some non-exist MDT-object, then the LFSCK should repair"
+ echo "the OST-object to back point to the right MDT-object."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to make the OST-object to back point to"
+ echo "non-exist MDT-object."
+ #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
+
+ do_facet ost1 $LCTL set_param fail_loc=0x1611
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
+ $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ $SHOW_LAYOUT
+ error "(2) unexpected status"
+ }
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+}
+run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
+
+test_15b() {
+ echo "#####"
+ echo "If the OST-object referenced by the MDT-object back points"
+ echo "to other MDT-object that doesn't recognize the OST-object,"
+ echo "then the LFSCK should repair it to back point to the right"
+ echo "MDT-object (the first one)."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+ dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
+ cancel_lru_locks osc
+
+ echo "Inject failure stub to make the OST-object to back point to"
+ echo "other MDT-object"
+
+ #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
+ do_facet ost1 $LCTL set_param fail_loc=0x1612
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
+ $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ $SHOW_LAYOUT
+ error "(2) unexpected status"
+ }
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+}
+run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
+
+test_16() {
+ echo "#####"
+ echo "If the OST-object's owner information does not match the owner"
+ echo "information stored in the MDT-object, then the LFSCK trust the"
+ echo "MDT-object and update the OST-object's owner information."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+
+ echo "Inject failure stub to skip OST-object owner changing"
+ #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
+ chown 1.1 $DIR/$tdir/f0
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
+ echo "and fix them"
+
+ $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ $SHOW_LAYOUT
+ error "(2) unexpected status"
+ }
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_inconsistent_owner/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair inconsistent owner: $repaired"
+}
+run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
+
+test_17() {
+ echo "#####"
+ echo "If more than one MDT-objects reference the same OST-object,"
+ echo "and the OST-object only recognizes one MDT-object, then the"
+ echo "LFSCK should create new OST-objects for such non-recognized"
+ echo "MDT-objects."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to make two MDT-objects to refernce"
+ echo "the OST-object"
+
+ #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0x1614
+
+ dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
+ cancel_lru_locks osc
+
+ createmany -o $DIR/$tdir/f 1
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
+ local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(1) f0 (wrong) size should be 1048576, but got $size"
+
+ echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
+ echo "and fix them"
+
+ $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 6 || {
+ $SHOW_LAYOUT
+ error "(3) unexpected status"
+ }
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_multiple_referenced/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to repair multiple references: $repaired"
+
+ echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
+ error "(5) Fail to write f0."
+ size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(6) guard size should be 1048576, but got $size"
+}
+run_test 17 "LFSCK can repair multiple references"
+
+test_18a() {
+ echo "#####"
+ echo "The target MDT-object is there, but related stripe information"
+ echo "is lost or partly lost. The LFSCK should regenerate the missed"
+ echo "layout EA entries."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
+
+ local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS mkdir -i 1 $DIR/$tdir/a2
+ $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
+ dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
+ $LFS path2fid $DIR/$tdir/a2/f2
+ $LFS getstripe $DIR/$tdir/a2/f2
+ fi
+
+ cancel_lru_locks osc
+
+ echo "Inject failure, to make the MDT-object lost its layout EA"
+ #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
+ do_facet mds1 $LCTL set_param fail_loc=0x1615
+ chown 1.1 $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0x1615
+ chown 1.1 $DIR/$tdir/a2/f2
+ fi
+
+ sync
+ sleep 2
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0
+ fi
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "The file size should be incorrect since layout EA is lost"
+ local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size" ] ||
+ error "(1) Expect incorrect file1 size"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size" ] ||
+ error "(2) Expect incorrect file2 size"
+ fi
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(4) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(5) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(6.1) Expect 1 fixed on mds1, but got: $repaired"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 2 ] ||
+ error "(6.2) Expect 2 fixed on mds2, but got: $repaired"
+ fi
+
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS path2fid $DIR/$tdir/a2/f2
+ $LFS getstripe $DIR/$tdir/a2/f2
+ fi
+
+ echo "The file size should be correct after layout LFSCK scanning"
+ cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(7) Expect file1 size $saved_size, but got $cur_size"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(8) Expect file2 size $saved_size, but got $cur_size"
+ fi
+}
+run_test 18a "Find out orphan OST-object and repair it (1)"
+
+test_18b() {
+ echo "#####"
+ echo "The target MDT-object is lost. The LFSCK should re-create the"
+ echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
+ echo "can move it back to normal namespace manually."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
+ local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
+ echo ${fid1}
+ $LFS getstripe $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS mkdir -i 1 $DIR/$tdir/a2
+ $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
+ dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
+ fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
+ echo ${fid2}
+ $LFS getstripe $DIR/$tdir/a2/f2
+ fi
+
+ cancel_lru_locks osc
+
+ echo "Inject failure, to simulate the case of missing the MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a2/f2
+ fi
+
+ sync
+ sleep 2
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0
+ fi
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(2) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(3) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4.1) Expect 1 fixed on mds1, but got: $repaired"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 2 ] ||
+ error "(4.2) Expect 2 fixed on mds2, but got: $repaired"
+ fi
+
+ echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
+ mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
+ error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ local name=$MOUNT/.lustre/lost+found/MDT0001/R-${fid2}
+ mv $name $DIR/$tdir/a2/f2 || error "(6) Fail to move $name"
+ fi
+
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS path2fid $DIR/$tdir/a2/f2
+ $LFS getstripe $DIR/$tdir/a2/f2
+ fi
+
+ echo "The file size should be correct after layout LFSCK scanning"
+ local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(7) Expect file1 size $saved_size, but got $cur_size"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(8) Expect file2 size $saved_size, but got $cur_size"
+ fi
+}
+run_test 18b "Find out orphan OST-object and repair it (2)"
+
+test_18c() {
+ echo "#####"
+ echo "The target MDT-object is lost, and the OST-object FID is missing."
+ echo "The LFSCK should re-create the MDT-object with new FID under the "
+ echo "directory .lustre/lost+found/MDTxxxx."
+ echo "#####"
+
+ check_mount_and_prep
+ $LFS mkdir -i 0 $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+
+ echo "Inject failure, to simulate the case of missing parent FID"
+ #define OBD_FAIL_LFSCK_NOPFID 0x1617
+ do_facet ost1 $LCTL set_param fail_loc=0x1617
+
+ dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
+ $LFS getstripe $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ $LFS mkdir -i 1 $DIR/$tdir/a2
+ $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
+ do_facet ost2 $LCTL set_param fail_loc=0x1617
+ dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
+ $LFS getstripe $DIR/$tdir/a2/f2
+ fi
+
+ cancel_lru_locks osc
+
+ echo "Inject failure, to simulate the case of missing the MDT-object"
+ #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
+ do_facet mds1 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a1/f1
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0x1616
+ rm -f $DIR/$tdir/a2/f2
+ fi
+
+ sync
+ sleep 2
+
+ do_facet mds1 $LCTL set_param fail_loc=0
+ if [ $MDSCOUNT -ge 2 ]; then
+ do_facet mds2 $LCTL set_param fail_loc=0
+ fi
+
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(2) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(3) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ expected=3
+ else
+ expected=1
+ fi
+
+ local repaired=$(do_facet mds1 $LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq $expected ] ||
+ error "(4) Expect $expected fixed on mds1, but got: $repaired"
+
+ if [ $MDSCOUNT -ge 2 ]; then
+ repaired=$(do_facet mds2 $LCTL get_param -n \
+ mdd.$(facet_svc mds2).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(5) Expect 0 fixed on mds2, but got: $repaired"
+ fi
+
+ echo "There should be some stub under .lustre/lost+found/MDT0001/"
+ ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
+ error "(6) .lustre/lost+found/MDT0001/ should be empty"
+
+ echo "There should be some stub under .lustre/lost+found/MDT0000/"
+ ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
+ error "(7) .lustre/lost+found/MDT0000/ should not be empty"
+}
+run_test 18c "Find out orphan OST-object and repair it (3)"
+
+test_18d() {
+ echo "#####"
+ echo "The target MDT-object layout EA slot is occpuied by some new"
+ echo "created OST-object when repair dangling reference case. Such"
+ echo "conflict OST-object has never been modified. Then when found"
+ echo "the orphan OST-object, LFSCK will replace it with the orphan"
+ echo "OST-object."
+ echo "#####"
+
+ check_mount_and_prep
+ mkdir $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+ echo "guard" > $DIR/$tdir/a1/f1
+ echo "foo" > $DIR/$tdir/a1/f2
+ local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+ $LFS path2fid $DIR/$tdir/a1/f2
+ $LFS getstripe $DIR/$tdir/a1/f2
+ cancel_lru_locks osc
+
+ echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
+ echo "to reference the same OST-object (which is f1's OST-obejct)."
+ echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
+ echo "dangling reference case, but f2's old OST-object is there."
+ echo
+
+ #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
+ chown 1.1 $DIR/$tdir/a1/f2
+ rm -f $DIR/$tdir/a1/f1
+ sync
+ sleep 2
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "The file size should be incorrect since dangling referenced"
+ local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size" ] ||
+ error "(1) Expect incorrect file2 size"
+
+ #define OBD_FAIL_LFSCK_DELAY3 0x1602
+ do_facet $SINGLEMDS $LCTL set_param fail_val=5 fail_loc=0x1602
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
+
+ wait_update_facet mds1 "$LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
+ error "(3.0) MDS1 is not the expected 'scanning-phase2'"
+
+ # LU-3469: before osp_sync() is enabled, wait for a while to guarantee
+ # that former async repair operations have been executed on the OST(s).
+ sync
+ sleep 2
+
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(3) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(4) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
+ mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(5) Expect 1 orphan has been fixed, but got: $repaired"
+
+ echo "The file size should be correct after layout LFSCK scanning"
+ cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(6) Expect file2 size $saved_size, but got $cur_size"
+
+ echo "The LFSCK should find back the original data."
+ cat $DIR/$tdir/a1/f2
+ $LFS path2fid $DIR/$tdir/a1/f2
+ $LFS getstripe $DIR/$tdir/a1/f2
+}
+run_test 18d "Find out orphan OST-object and repair it (4)"
+
+test_18e() {
+ echo "#####"
+ echo "The target MDT-object layout EA slot is occpuied by some new"
+ echo "created OST-object when repair dangling reference case. Such"
+ echo "conflict OST-object has been modified by others. To keep the"
+ echo "new data, the LFSCK will create a new file to refernece this"
+ echo "old orphan OST-object."
+ echo "#####"
+
+ check_mount_and_prep
+ mkdir $DIR/$tdir/a1
+ $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
+ echo "guard" > $DIR/$tdir/a1/f1
+ echo "foo" > $DIR/$tdir/a1/f2
+ local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ $LFS path2fid $DIR/$tdir/a1/f1
+ $LFS getstripe $DIR/$tdir/a1/f1
+ $LFS path2fid $DIR/$tdir/a1/f2
+ $LFS getstripe $DIR/$tdir/a1/f2
+ cancel_lru_locks osc
+
+ echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
+ echo "to reference the same OST-object (which is f1's OST-obejct)."
+ echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
+ echo "dangling reference case, but f2's old OST-object is there."
+ echo
+
+ #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
+ chown 1.1 $DIR/$tdir/a1/f2
+ rm -f $DIR/$tdir/a1/f1
+ sync
+ sleep 2
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "The file size should be incorrect since dangling referenced"
+ local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
+ [ "$cur_size" != "$saved_size" ] ||
+ error "(1) Expect incorrect file2 size"
+
+ #define OBD_FAIL_LFSCK_DELAY3 0x1602
+ do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
+
+ echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
+ $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
+
+ wait_update_facet mds1 "$LCTL get_param -n \
+ mdd.$(facet_svc mds1).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
+ error "(3) MDS1 is not the expected 'scanning-phase2'"
+
+ # to guarantee all updates are synced.
+ sync
+ sleep 2
+
+ echo "Write new data to f2 to modify the new created OST-object."
+ echo "dummy" >> $DIR/$tdir/a1/f2
+
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
+
+ for k in $(seq $MDSCOUNT); do
+ # The LFSCK status query internal is 30 seconds. For the case
+ # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
+ # time to guarantee the status sync up.
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 32 ||
+ error "(4) MDS${k} is not the expected 'completed'"
+ done
+
+ for k in $(seq $OSTCOUNT); do
+ local cur_status=$(do_facet ost${k} $LCTL get_param -n \
+ obdfilter.$(facet_svc ost${k}).lfsck_layout |
+ awk '/^status/ { print $2 }')
+ [ "$cur_status" == "completed" ] ||
+ error "(5) OST${k} Expect 'completed', but got '$cur_status'"
+ done
+
+ local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
+ mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
+ awk '/^repaired_orphan/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(6) Expect 1 orphan has been fixed, but got: $repaired"
+
+ echo "There should be stub file under .lustre/lost+found/MDT0000/"
+ local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
+ [ ! -z $name ] ||
+ error "(7) .lustre/lost+found/MDT0000/ should not be empty"
+
+ echo "The stub file should keep the original f2 data"
+ cur_size=$(ls -il $cname | awk '{ print $6 }')
+ [ "$cur_size" == "$saved_size" ] ||
+ error "(8) Expect file2 size $saved_size, but got $cur_size"
+
+ cat $cname
+ $LFS path2fid $cname
+ $LFS getstripe $cname
+
+ echo "The f2 should contains new data."
+ cat $DIR/$tdir/a1/f2
+ $LFS path2fid $DIR/$tdir/a1/f2
+ $LFS getstripe $DIR/$tdir/a1/f2
+}
+run_test 18e "Find out orphan OST-object and repair it (5)"
+
+test_19a() {
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "foo" > $DIR/$tdir/a0
+ echo "guard" > $DIR/$tdir/a1
+ cancel_lru_locks osc
+
+ echo "Inject failure, then client will offer wrong parent FID when read"
+ do_facet ost1 $LCTL set_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
+ #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
+ $LCTL set_param fail_loc=0x1619
+
+ echo "Read RPC with wrong parent FID should be denied"
+ cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
+ $LCTL set_param fail_loc=0
+}
+run_test 19a "OST-object inconsistency self detect"
+
+test_19b() {
+ check_mount_and_prep
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to make the OST-object to back point to"
+ echo "non-exist MDT-object"
+
+ #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
+ do_facet ost1 $LCTL set_param fail_loc=0x1611
+ echo "foo" > $DIR/$tdir/f0
+ cancel_lru_locks osc
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "Nothing should be fixed since self detect and repair is disabled"
+ local repaired=$(do_facet ost1 $LCTL get_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
+ awk '/^repaired/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(1) Expected 0 repaired, but got $repaired"
+
+ echo "Read RPC with right parent FID should be accepted,"
+ echo "and cause parent FID on OST to be fixed"
+
+ do_facet ost1 $LCTL set_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
+ cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
+
+ repaired=$(do_facet ost1 $LCTL get_param -n \
+ obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
+ awk '/^repaired/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Expected 1 repaired, but got $repaired"
+}
+run_test 19b "OST-object inconsistency self repair"
+