+#!/bin/bash
+#
+# Run select tests by setting ONLY, or as arguments to the script.
+# Skip specific tests by setting EXCEPT.
+#
+
+set -e
+
+ONLY=${ONLY:-"$*"}
+ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
+[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
+# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
+
+LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
+
+[ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
+ skip "test LFSCK only for ldiskfs" && exit 0
+require_dsh_mds || exit 0
+
+SAVED_MDSSIZE=${MDSSIZE}
+SAVED_OSTSIZE=${OSTSIZE}
+# use small MDS + OST size to speed formatting time
+# do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
+MDSSIZE=100000
+OSTSIZE=100000
+
+check_and_setup_lustre
+build_test_filter
+
+$LCTL set_param debug=+lfsck > /dev/null || true
+
+MDT_DEV="${FSNAME}-MDT0000"
+MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
+START_NAMESPACE="do_facet $SINGLEMDS \
+ $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
+STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
+SHOW_NAMESPACE="do_facet $SINGLEMDS \
+ $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
+MOUNT_OPTS_SCRUB="-o user_xattr"
+MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
+
+lfsck_prep() {
+ local ndirs=$1
+ local nfiles=$2
+
+ echo "formatall"
+ formatall > /dev/null
+
+ echo "setupall"
+ setupall > /dev/null
+
+ echo "preparing... ${nfiles} * ${ndirs} files will be created."
+ mkdir -p $DIR/$tdir
+ cp $LUSTRE/tests/*.sh $DIR/$tdir/
+ for ((i=0; i<${ndirs}; i++)); do
+ mkdir $DIR/$tdir/d${i}
+ touch $DIR/$tdir/f${i}
+ for ((j=0; j<${nfiles}; j++)); do
+ touch $DIR/$tdir/d${i}/f${j}
+ done
+ mkdir $DIR/$tdir/e${i}
+ done
+
+ echo "prepared."
+ cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
+}
+
+test_0() {
+ lfsck_prep 10 10
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ #define OBD_FAIL_LFSCK_DELAY1 0x1600
+ do_facet $SINGLEMDS $LCTL set_param fail_val=3
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
+ $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
+
+ $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(4) Expect 'scanning-phase1', but got '$STATUS'"
+
+ $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "stopped" ] ||
+ error "(6) Expect 'stopped', but got '$STATUS'"
+
+ $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(8) Expect 'scanning-phase1', but got '$STATUS'"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(9) Expect 'completed', but got '$STATUS'"
+
+ local repaired=$($SHOW_NAMESPACE |
+ awk '/^updated_phase1/ { print $2 }')
+ [ $repaired -eq 0 ] ||
+ error "(10) Expect nothing to be repaired, but got: $repaired"
+}
+run_test 0 "Control LFSCK manually"
+
+test_6a() {
+ lfsck_prep 10 10
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ #define OBD_FAIL_LFSCK_DELAY1 0x1600
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
+ $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ # Sleep 3 sec to guarantee at least one object processed by LFSCK
+ sleep 3
+ # Fail the LFSCK to guarantee there is at least one checkpoint
+ #define OBD_FAIL_LFSCK_FATAL1 0x1608
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "failed" ] ||
+ error "(4) Expect 'failed', but got '$STATUS'"
+
+ local POSITION0=$($SHOW_NAMESPACE |
+ awk '/^last_checkpoint_position/ { print $2 }' |
+ tr -d ',')
+
+ #define OBD_FAIL_LFSCK_DELAY1 0x1600
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
+ $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(6) Expect 'scanning-phase1', but got '$STATUS'"
+
+ local POSITION1=$($SHOW_NAMESPACE |
+ awk '/^latest_start_position/ { print $2 }' |
+ tr -d ',')
+ [ $POSITION0 -lt $POSITION1 ] ||
+ error "(7) Expect larger than: $POSITION0, but got $POSITION1"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(8) Expect 'completed', but got '$STATUS'"
+}
+run_test 6a "LFSCK resumes from last checkpoint (1)"
+
+test_6b() {
+ lfsck_prep 10 10
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ #define OBD_FAIL_LFSCK_DELAY2 0x1601
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
+ $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ # Sleep 3 sec to guarantee at least one object processed by LFSCK
+ sleep 3
+ # Fail the LFSCK to guarantee there is at least one checkpoint
+ #define OBD_FAIL_LFSCK_FATAL2 0x1609
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "failed" ] ||
+ error "(4) Expect 'failed', but got '$STATUS'"
+
+ local POSITION0=$($SHOW_NAMESPACE |
+ awk '/^last_checkpoint_position/ { print $4 }')
+
+ #define OBD_FAIL_LFSCK_DELAY2 0x1601
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
+ $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(6) Expect 'scanning-phase1', but got '$STATUS'"
+
+ local POSITION1=$($SHOW_NAMESPACE |
+ awk '/^latest_start_position/ { print $4 }')
+ if [ $POSITION0 -gt $POSITION1 ]; then
+ [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] ||
+ error "(7) Expect larger than: $POSITION0, but got $POSITION1"
+ fi
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(8) Expect 'completed', but got '$STATUS'"
+}
+run_test 6b "LFSCK resumes from last checkpoint (2)"
+
+test_7a()
+{
+ lfsck_prep 10 10
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ #define OBD_FAIL_LFSCK_DELAY2 0x1601
+ do_facet $SINGLEMDS $LCTL set_param fail_val=1
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
+ $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ # Sleep 3 sec to guarantee at least one object processed by LFSCK
+ sleep 3
+ echo "stop $SINGLEMDS"
+ stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
+
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(5) Fail to start MDS!"
+
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(6) Expect 'scanning-phase1', but got '$STATUS'"
+
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_val=0
+ sleep 3
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(7) Expect 'completed', but got '$STATUS'"
+}
+run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
+
+test_9a() {
+ if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
+ skip "Testing on UP system, the speed may be inaccurate."
+ return 0
+ fi
+
+ lfsck_prep 70 70
+ echo "start $SINGLEMDS"
+ start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
+ error "(1) Fail to start MDS!"
+
+ local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "init" ] ||
+ error "(2) Expect 'init', but got '$STATUS'"
+
+ $START_NAMESPACE -s 100 || error "(3) Fail to start LFSCK!"
+
+ sleep 10
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "scanning-phase1" ] ||
+ error "(3) Expect 'scanning-phase1', but got '$STATUS'"
+
+ local SPEED=$($SHOW_NAMESPACE |
+ awk '/^average_speed_phase1/ { print $2 }')
+ # (100 * (10 + 1)) / 10 = 110
+ [ $SPEED -lt 120 ] ||
+ error "(4) Unexpected speed $SPEED, should not more than 120"
+
+ # adjust speed limit
+ do_facet $SINGLEMDS \
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
+ sleep 10
+
+ SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
+ # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180
+ [ $SPEED -lt 170 ] &&
+ error "(5) Unexpected speed $SPEED, should not less than 170"
+
+ # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220
+ [ $SPEED -lt 230 ] ||
+ error "(6) Unexpected speed $SPEED, should not more than 230"
+
+ do_facet $SINGLEMDS \
+ $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
+ sleep 5
+ STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
+ [ "$STATUS" == "completed" ] ||
+ error "(7) Expect 'completed', but got '$STATUS'"
+}
+run_test 9a "LFSCK speed control (1)"
+
+$LCTL set_param debug=-lfsck > /dev/null || true
+
+# restore MDS/OST size
+MDSSIZE=${SAVED_MDSSIZE}
+OSTSIZE=${SAVED_OSTSIZE}
+
+# cleanup the system at last
+formatall
+
+complete $SECONDS
+exit_status
git://git.whamcloud.com / fs / lustre-release.git / commitdiff