3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
21 skip "test LFSCK only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 SAVED_MDSSIZE=${MDSSIZE}
25 SAVED_OSTSIZE=${OSTSIZE}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
31 check_and_setup_lustre
34 $LCTL set_param debug=+lfsck > /dev/null || true
36 MDT_DEV="${FSNAME}-MDT0000"
37 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
38 START_NAMESPACE="do_facet $SINGLEMDS \
39 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
40 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
41 SHOW_NAMESPACE="do_facet $SINGLEMDS \
42 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MOUNT_OPTS_SCRUB="-o user_xattr"
44 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
56 echo "preparing... ${nfiles} * ${ndirs} files will be created."
58 cp $LUSTRE/tests/*.sh $DIR/$tdir/
59 for ((i=0; i<${ndirs}; i++)); do
60 mkdir $DIR/$tdir/d${i}
61 touch $DIR/$tdir/f${i}
62 for ((j=0; j<${nfiles}; j++)); do
63 touch $DIR/$tdir/d${i}/f${j}
65 mkdir $DIR/$tdir/e${i}
69 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
70 echo "stop $SINGLEMDS"
71 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
76 echo "start $SINGLEMDS"
77 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
78 error "(1) Fail to start MDS!"
80 #define OBD_FAIL_LFSCK_DELAY1 0x1600
81 do_facet $SINGLEMDS $LCTL set_param fail_val=3
82 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
83 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
85 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
87 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
88 [ "$STATUS" == "scanning-phase1" ] ||
89 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
91 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
93 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
94 [ "$STATUS" == "stopped" ] ||
95 error "(6) Expect 'stopped', but got '$STATUS'"
97 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
99 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
100 [ "$STATUS" == "scanning-phase1" ] ||
101 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
103 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
104 do_facet $SINGLEMDS $LCTL set_param fail_val=0
106 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
107 [ "$STATUS" == "completed" ] ||
108 error "(9) Expect 'completed', but got '$STATUS'"
110 local repaired=$($SHOW_NAMESPACE |
111 awk '/^updated_phase1/ { print $2 }')
112 [ $repaired -eq 0 ] ||
113 error "(10) Expect nothing to be repaired, but got: $repaired"
115 run_test 0 "Control LFSCK manually"
119 echo "start $SINGLEMDS"
120 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
121 error "(1) Fail to start MDS!"
123 #define OBD_FAIL_LFSCK_DELAY1 0x1600
124 do_facet $SINGLEMDS $LCTL set_param fail_val=1
125 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
126 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
128 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
129 [ "$STATUS" == "scanning-phase1" ] ||
130 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
132 # Sleep 3 sec to guarantee at least one object processed by LFSCK
134 # Fail the LFSCK to guarantee there is at least one checkpoint
135 #define OBD_FAIL_LFSCK_FATAL1 0x1608
136 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
138 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
139 [ "$STATUS" == "failed" ] ||
140 error "(4) Expect 'failed', but got '$STATUS'"
142 local POSITION0=$($SHOW_NAMESPACE |
143 awk '/^last_checkpoint_position/ { print $2 }' |
146 #define OBD_FAIL_LFSCK_DELAY1 0x1600
147 do_facet $SINGLEMDS $LCTL set_param fail_val=1
148 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
149 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
151 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
152 [ "$STATUS" == "scanning-phase1" ] ||
153 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
155 local POSITION1=$($SHOW_NAMESPACE |
156 awk '/^latest_start_position/ { print $2 }' |
158 [ $POSITION0 -lt $POSITION1 ] ||
159 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
161 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
162 do_facet $SINGLEMDS $LCTL set_param fail_val=0
164 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
165 [ "$STATUS" == "completed" ] ||
166 error "(8) Expect 'completed', but got '$STATUS'"
168 run_test 6a "LFSCK resumes from last checkpoint (1)"
172 echo "start $SINGLEMDS"
173 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
174 error "(1) Fail to start MDS!"
176 #define OBD_FAIL_LFSCK_DELAY2 0x1601
177 do_facet $SINGLEMDS $LCTL set_param fail_val=1
178 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
179 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
181 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
182 [ "$STATUS" == "scanning-phase1" ] ||
183 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
185 # Sleep 3 sec to guarantee at least one object processed by LFSCK
187 # Fail the LFSCK to guarantee there is at least one checkpoint
188 #define OBD_FAIL_LFSCK_FATAL2 0x1609
189 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
191 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
192 [ "$STATUS" == "failed" ] ||
193 error "(4) Expect 'failed', but got '$STATUS'"
195 local POSITION0=$($SHOW_NAMESPACE |
196 awk '/^last_checkpoint_position/ { print $4 }')
198 #define OBD_FAIL_LFSCK_DELAY2 0x1601
199 do_facet $SINGLEMDS $LCTL set_param fail_val=1
200 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
201 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
203 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
204 [ "$STATUS" == "scanning-phase1" ] ||
205 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
207 local POSITION1=$($SHOW_NAMESPACE |
208 awk '/^latest_start_position/ { print $4 }')
209 if [ $POSITION0 -gt $POSITION1 ]; then
210 [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] ||
211 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
214 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
215 do_facet $SINGLEMDS $LCTL set_param fail_val=0
217 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
218 [ "$STATUS" == "completed" ] ||
219 error "(8) Expect 'completed', but got '$STATUS'"
221 run_test 6b "LFSCK resumes from last checkpoint (2)"
226 echo "start $SINGLEMDS"
227 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
228 error "(1) Fail to start MDS!"
230 #define OBD_FAIL_LFSCK_DELAY2 0x1601
231 do_facet $SINGLEMDS $LCTL set_param fail_val=1
232 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
233 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
235 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
236 [ "$STATUS" == "scanning-phase1" ] ||
237 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
239 # Sleep 3 sec to guarantee at least one object processed by LFSCK
241 echo "stop $SINGLEMDS"
242 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
244 echo "start $SINGLEMDS"
245 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
246 error "(5) Fail to start MDS!"
248 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
249 [ "$STATUS" == "scanning-phase1" ] ||
250 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
252 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
253 do_facet $SINGLEMDS $LCTL set_param fail_val=0
255 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
256 [ "$STATUS" == "completed" ] ||
257 error "(7) Expect 'completed', but got '$STATUS'"
259 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
262 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
263 skip "Testing on UP system, the speed may be inaccurate."
268 echo "start $SINGLEMDS"
269 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
270 error "(1) Fail to start MDS!"
272 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
273 [ "$STATUS" == "init" ] ||
274 error "(2) Expect 'init', but got '$STATUS'"
276 $START_NAMESPACE -s 100 || error "(3) Fail to start LFSCK!"
279 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
280 [ "$STATUS" == "scanning-phase1" ] ||
281 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
283 local SPEED=$($SHOW_NAMESPACE |
284 awk '/^average_speed_phase1/ { print $2 }')
285 # (100 * (10 + 1)) / 10 = 110
286 [ $SPEED -lt 120 ] ||
287 error "(4) Unexpected speed $SPEED, should not more than 120"
290 do_facet $SINGLEMDS \
291 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
294 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
295 # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180
296 [ $SPEED -lt 170 ] &&
297 error "(5) Unexpected speed $SPEED, should not less than 170"
299 # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220
300 [ $SPEED -lt 230 ] ||
301 error "(6) Unexpected speed $SPEED, should not more than 230"
303 do_facet $SINGLEMDS \
304 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
306 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
307 [ "$STATUS" == "completed" ] ||
308 error "(7) Expect 'completed', but got '$STATUS'"
310 run_test 9a "LFSCK speed control (1)"
312 $LCTL set_param debug=-lfsck > /dev/null || true
314 # restore MDS/OST size
315 MDSSIZE=${SAVED_MDSSIZE}
316 OSTSIZE=${SAVED_OSTSIZE}
318 # cleanup the system at last