[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
-[[ $(lustre_version_code ost1) -lt $(version_code 2.5.50) ]] &&
- ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14"
+[[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17"
build_test_filter
echo "setupall"
setupall > /dev/null
+ mkdir -p $DIR/$tdir
+
echo "All the LFSCK targets should be in 'init' status."
for k in $(seq $MDSCOUNT); do
local STATUS=$(do_facet mds${k} $LCTL get_param -n \
[ "$STATUS" == "init" ] ||
error "(1) MDS${k} Expect 'init', but got '$STATUS'"
- $LFS mkdir -i $((k - 1)) $DIR/${k}
- createmany -o $DIR/${k}/f 100
+ $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
+ createmany -o $DIR/$tdir/${k}/f 100
+ done
+
+ echo "Start namespace LFSCK on all targets by single command (-s 1)."
+ do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
+ -s 1 || error "(2) Fail to start LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'scanning-phase1' status."
+ for k in $(seq $MDSCOUNT); do
+ local STATUS=$(do_facet mds${k} $LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_namespace |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
+ done
+
+ echo "Stop namespace LFSCK on all targets by single lctl command."
+ do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
+ error "(4) Fail to stop LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'stopped' status."
+ for k in $(seq $MDSCOUNT); do
+ local STATUS=$(do_facet mds${k} $LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_namespace |
+ awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
+ done
+
+ echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
+ do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
+ -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
+
+ echo "All the LFSCK targets should be in 'completed' status."
+ for k in $(seq $MDSCOUNT); do
+ wait_update_facet mds${k} "$LCTL get_param -n \
+ mdd.$(facet_svc mds${k}).lfsck_namespace |
+ awk '/^status/ { print \\\$2 }'" "completed" 8 ||
+ error "(7) MDS${k} is not the expected 'completed'"
done
- echo "Trigger LFSCK on all targets by single command (limited speed)."
+ echo "Start layout LFSCK on all targets by single command (-s 1)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
- -s 10 || error "(2) Fail to start LFSCK on all devices!"
+ -s 1 || error "(8) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'scanning-phase1' status."
for k in $(seq $MDSCOUNT); do
mdd.$(facet_svc mds${k}).lfsck_layout |
awk '/^status/ { print $2 }')
[ "$STATUS" == "scanning-phase1" ] ||
- error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
+ error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
done
echo "Stop layout LFSCK on all targets by single lctl command."
do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
- error "(4) Fail to stop LFSCK on all devices!"
+ error "(10) Fail to stop LFSCK on all devices!"
echo "All the LFSCK targets should be in 'stopped' status."
for k in $(seq $MDSCOUNT); do
mdd.$(facet_svc mds${k}).lfsck_layout |
awk '/^status/ { print $2 }')
[ "$STATUS" == "stopped" ] ||
- error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
+ error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
done
for k in $(seq $OSTCOUNT); do
obdfilter.$(facet_svc ost${k}).lfsck_layout |
awk '/^status/ { print $2 }')
[ "$STATUS" == "stopped" ] ||
- error "(6) OST${k} Expect 'stopped', but got '$STATUS'"
+ error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
done
- echo "Re-trigger LFSCK on all targets by single command (full speed)."
+ echo "Re-start layout LFSCK on all targets by single command (-s 0)."
do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
- -s 0 || error "(7) Fail to start LFSCK on all devices!"
+ -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
echo "All the LFSCK targets should be in 'completed' status."
for k in $(seq $MDSCOUNT); do
wait_update_facet mds${k} "$LCTL get_param -n \
mdd.$(facet_svc mds${k}).lfsck_layout |
awk '/^status/ { print \\\$2 }'" "completed" 32 ||
- error "(8) MDS${k} is not the expected 'completed'"
+ error "(14) MDS${k} is not the expected 'completed'"
done
}
run_test 12 "single command to trigger LFSCK on all devices"
}
run_test 14 "LFSCK can repair MDT-object with dangling reference"
+test_15a() {
+ echo "#####"
+ echo "If the OST-object referenced by the MDT-object back points"
+ echo "to some non-exist MDT-object, then the LFSCK should repair"
+ echo "the OST-object to back point to the right MDT-object."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to make the OST-object to back point to"
+ echo "non-exist MDT-object."
+ #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
+
+ do_facet ost1 $LCTL set_param fail_loc=0x1611
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+ sync
+ sleep 2
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
+ $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+}
+run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
+
+test_15b() {
+ echo "#####"
+ echo "If the OST-object referenced by the MDT-object back points"
+ echo "to other MDT-object that doesn't recognize the OST-object,"
+ echo "then the LFSCK should repair it to back point to the right"
+ echo "MDT-object (the first one)."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+ touch $DIR/$tdir/guard
+
+ echo "Inject failure stub to make the OST-object to back point to"
+ echo "other MDT-object"
+
+ #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
+ do_facet ost1 $LCTL set_param fail_loc=0x1612
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+ sync
+ sleep 2
+ do_facet ost1 $LCTL set_param fail_loc=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
+ $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_unmatched_pair/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair unmatched pair: $repaired"
+}
+run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
+
+test_16() {
+ echo "#####"
+ echo "If the OST-object's owner information does not match the owner"
+ echo "information stored in the MDT-object, then the LFSCK trust the"
+ echo "MDT-object and update the OST-object's owner information."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
+ cancel_lru_locks osc
+ sync
+ sleep 2
+
+ echo "Inject failure stub to skip OST-object owner changing"
+ #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
+ chown 1.1 $DIR/$tdir/f0
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+ echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
+ echo "and fix them"
+
+ $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_inconsistent_owner/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(3) Fail to repair inconsistent owner: $repaired"
+}
+run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
+
+test_17() {
+ echo "#####"
+ echo "If more than one MDT-objects reference the same OST-object,"
+ echo "and the OST-object only recognizes one MDT-object, then the"
+ echo "LFSCK should create new OST-objects for such non-recognized"
+ echo "MDT-objects."
+ echo "#####"
+
+ echo "stopall"
+ stopall > /dev/null
+ echo "formatall"
+ formatall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ mkdir -p $DIR/$tdir
+ $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+ echo "Inject failure stub to make two MDT-objects to refernce"
+ echo "the OST-object"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1614
+
+ dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
+ cancel_lru_locks osc
+ sync
+ sleep 2
+
+ createmany -o $DIR/$tdir/f 1 > /dev/null 2>&1
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+
+ echo "stopall to cleanup object cache"
+ stopall > /dev/null
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
+ local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(1) f0 (wrong) size should be 1048576, but got $size"
+
+ echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
+ echo "and fix them"
+
+ $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
+
+ wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+ mdd.${MDT_DEV}.lfsck_layout |
+ awk '/^status/ { print \\\$2 }'" "completed" 3 || return 3
+
+ local repaired=$($SHOW_LAYOUT |
+ awk '/^repaired_multiple_referenced/ { print $2 }')
+ [ $repaired -eq 1 ] ||
+ error "(4) Fail to repair multiple references: $repaired"
+
+ echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
+ dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
+ error "(5) Fail to write f0."
+ size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
+ [ $size -eq 1048576 ] ||
+ error "(6) guard size should be 1048576, but got $size"
+}
+run_test 17 "LFSCK can repair multiple references"
+
$LCTL set_param debug=-lfsck > /dev/null || true
# restore MDS/OST size
git://git.whamcloud.com / fs / lustre-release.git / blobdiff