3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
21 skip "test LFSCK only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 MCREATE=${MCREATE:-mcreate}
25 SAVED_MDSSIZE=${MDSSIZE}
26 SAVED_OSTSIZE=${OSTSIZE}
27 # use small MDS + OST size to speed formatting time
28 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
32 check_and_setup_lustre
35 $LCTL set_param debug=+lfsck > /dev/null || true
37 MDT_DEV="${FSNAME}-MDT0000"
38 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
39 START_NAMESPACE="do_facet $SINGLEMDS \
40 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
42 SHOW_NAMESPACE="do_facet $SINGLEMDS \
43 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
44 MOUNT_OPTS_SCRUB="-o user_xattr"
45 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
55 if [ ! -z $igif ]; then
56 #define OBD_FAIL_FID_IGIF 0x1504
57 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
63 echo "preparing... ${nfiles} * ${ndirs} files will be created."
65 cp $LUSTRE/tests/*.sh $DIR/$tdir/
66 for ((i=0; i<${ndirs}; i++)); do
67 mkdir $DIR/$tdir/d${i}
68 touch $DIR/$tdir/f${i}
69 for ((j=0; j<${nfiles}; j++)); do
70 touch $DIR/$tdir/d${i}/f${j}
72 mkdir $DIR/$tdir/e${i}
75 if [ ! -z $igif ]; then
76 touch $DIR/$tdir/dummy
77 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
81 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
82 echo "stop $SINGLEMDS"
83 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
88 echo "start $SINGLEMDS"
89 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
90 error "(1) Fail to start MDS!"
92 #define OBD_FAIL_LFSCK_DELAY1 0x1600
93 do_facet $SINGLEMDS $LCTL set_param fail_val=3
94 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
95 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
97 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
99 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
100 [ "$STATUS" == "scanning-phase1" ] ||
101 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
103 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
105 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
106 [ "$STATUS" == "stopped" ] ||
107 error "(6) Expect 'stopped', but got '$STATUS'"
109 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
111 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
112 [ "$STATUS" == "scanning-phase1" ] ||
113 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
115 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
116 do_facet $SINGLEMDS $LCTL set_param fail_val=0
118 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
119 [ "$STATUS" == "completed" ] ||
120 error "(9) Expect 'completed', but got '$STATUS'"
122 local repaired=$($SHOW_NAMESPACE |
123 awk '/^updated_phase1/ { print $2 }')
124 [ $repaired -eq 0 ] ||
125 error "(10) Expect nothing to be repaired, but got: $repaired"
127 run_test 0 "Control LFSCK manually"
131 echo "start $SINGLEMDS"
132 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
133 error "(1) Fail to start MDS!"
135 mount_client $MOUNT || error "(2) Fail to start client!"
137 #define OBD_FAIL_FID_INDIR 0x1501
138 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
139 touch $DIR/$tdir/dummy
141 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
143 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
146 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
147 [ "$STATUS" == "completed" ] ||
148 error "(4) Expect 'completed', but got '$STATUS'"
150 local repaired=$($SHOW_NAMESPACE |
151 awk '/^updated_phase1/ { print $2 }')
152 [ $repaired -eq 1 ] ||
153 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
155 mount_client $MOUNT || error "(6) Fail to start client!"
157 #define OBD_FAIL_FID_LOOKUP 0x1505
158 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
159 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
161 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
163 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
168 echo "start $SINGLEMDS"
169 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
170 error "(1) Fail to start MDS!"
172 mount_client $MOUNT || error "(2) Fail to start client!"
174 #define OBD_FAIL_FID_INLMA 0x1502
175 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
176 touch $DIR/$tdir/dummy
178 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
180 #define OBD_FAIL_FID_NOLMA 0x1506
181 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
182 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
185 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
186 [ "$STATUS" == "completed" ] ||
187 error "(4) Expect 'completed', but got '$STATUS'"
189 local repaired=$($SHOW_NAMESPACE |
190 awk '/^updated_phase1/ { print $2 }')
191 [ $repaired -eq 1 ] ||
192 error "(5) Fail to repair missed FID-in-LMA: $repaired"
194 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
195 mount_client $MOUNT || error "(6) Fail to start client!"
197 #define OBD_FAIL_FID_LOOKUP 0x1505
198 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
199 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
201 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
203 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
208 mds_backup_restore || error "(1) Fail to backup/restore!"
209 echo "start $SINGLEMDS with disabling OI scrub"
210 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
211 error "(2) Fail to start MDS!"
213 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
214 [ "$STATUS" == "init" ] ||
215 error "(3) Expect 'init', but got '$STATUS'"
217 #define OBD_FAIL_LFSCK_DELAY2 0x1601
218 do_facet $SINGLEMDS $LCTL set_param fail_val=1
219 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
220 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
223 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
224 [ "$STATUS" == "scanning-phase1" ] ||
225 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
227 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
228 [ "$FLAGS" == "inconsistent" ] ||
229 error "(6) Expect 'inconsistent', but got '$FLAGS'"
231 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
232 do_facet $SINGLEMDS $LCTL set_param fail_val=0
234 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
235 [ "$STATUS" == "completed" ] ||
236 error "(7) Expect 'completed', but got '$STATUS'"
238 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
239 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
241 local repaired=$($SHOW_NAMESPACE |
242 awk '/^updated_phase1/ { print $2 }')
243 [ $repaired -ge 9 ] ||
244 error "(9) Fail to repair crashed linkEA: $repaired"
246 mount_client $MOUNT || error "(10) Fail to start client!"
248 #define OBD_FAIL_FID_LOOKUP 0x1505
249 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
250 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
252 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
254 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
259 mds_backup_restore 1 || error "(1) Fail to backup/restore!"
260 echo "start $SINGLEMDS with disabling OI scrub"
261 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
262 error "(2) Fail to start MDS!"
264 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
265 [ "$STATUS" == "init" ] ||
266 error "(3) Expect 'init', but got '$STATUS'"
268 #define OBD_FAIL_LFSCK_DELAY2 0x1601
269 do_facet $SINGLEMDS $LCTL set_param fail_val=1
270 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
271 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
274 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
275 [ "$STATUS" == "scanning-phase1" ] ||
276 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
278 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
279 [ "$FLAGS" == "inconsistent,upgrade" ] ||
280 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
282 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
283 do_facet $SINGLEMDS $LCTL set_param fail_val=0
285 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
286 [ "$STATUS" == "completed" ] ||
287 error "(7) Expect 'completed', but got '$STATUS'"
289 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
290 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
292 local repaired=$($SHOW_NAMESPACE |
293 awk '/^updated_phase1/ { print $2 }')
294 [ $repaired -ge 2 ] ||
295 error "(9) Fail to repair crashed linkEA: $repaired"
297 mount_client $MOUNT || error "(10) Fail to start client!"
299 #define OBD_FAIL_FID_LOOKUP 0x1505
300 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
301 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
303 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
305 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
306 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
307 local dummyname=$($LFS fid2path $DIR $dummyfid)
308 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
309 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
311 run_test 5 "LFSCK can handle IFIG object upgrading"
315 echo "start $SINGLEMDS"
316 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
317 error "(1) Fail to start MDS!"
319 #define OBD_FAIL_LFSCK_DELAY1 0x1600
320 do_facet $SINGLEMDS $LCTL set_param fail_val=1
321 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
322 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
324 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
325 [ "$STATUS" == "scanning-phase1" ] ||
326 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
328 # Sleep 3 sec to guarantee at least one object processed by LFSCK
330 # Fail the LFSCK to guarantee there is at least one checkpoint
331 #define OBD_FAIL_LFSCK_FATAL1 0x1608
332 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
334 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
335 [ "$STATUS" == "failed" ] ||
336 error "(4) Expect 'failed', but got '$STATUS'"
338 local POSITION0=$($SHOW_NAMESPACE |
339 awk '/^last_checkpoint_position/ { print $2 }' |
342 #define OBD_FAIL_LFSCK_DELAY1 0x1600
343 do_facet $SINGLEMDS $LCTL set_param fail_val=1
344 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
345 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
347 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
348 [ "$STATUS" == "scanning-phase1" ] ||
349 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
351 local POSITION1=$($SHOW_NAMESPACE |
352 awk '/^latest_start_position/ { print $2 }' |
354 [ $POSITION0 -lt $POSITION1 ] ||
355 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
357 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
358 do_facet $SINGLEMDS $LCTL set_param fail_val=0
360 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
361 [ "$STATUS" == "completed" ] ||
362 error "(8) Expect 'completed', but got '$STATUS'"
364 run_test 6a "LFSCK resumes from last checkpoint (1)"
368 echo "start $SINGLEMDS"
369 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
370 error "(1) Fail to start MDS!"
372 #define OBD_FAIL_LFSCK_DELAY2 0x1601
373 do_facet $SINGLEMDS $LCTL set_param fail_val=1
374 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
375 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
377 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
378 [ "$STATUS" == "scanning-phase1" ] ||
379 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
381 # Sleep 3 sec to guarantee at least one object processed by LFSCK
383 # Fail the LFSCK to guarantee there is at least one checkpoint
384 #define OBD_FAIL_LFSCK_FATAL2 0x1609
385 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
387 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
388 [ "$STATUS" == "failed" ] ||
389 error "(4) Expect 'failed', but got '$STATUS'"
391 local POSITION0=$($SHOW_NAMESPACE |
392 awk '/^last_checkpoint_position/ { print $4 }')
394 #define OBD_FAIL_LFSCK_DELAY2 0x1601
395 do_facet $SINGLEMDS $LCTL set_param fail_val=1
396 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
397 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
399 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
400 [ "$STATUS" == "scanning-phase1" ] ||
401 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
403 local POSITION1=$($SHOW_NAMESPACE |
404 awk '/^latest_start_position/ { print $4 }')
405 if [ $POSITION0 -gt $POSITION1 ]; then
406 [ $POSITION1 -eq 0 -a $POSITINO0 -eq $((POSITION1 + 1)) ] ||
407 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
410 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
411 do_facet $SINGLEMDS $LCTL set_param fail_val=0
413 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
414 [ "$STATUS" == "completed" ] ||
415 error "(8) Expect 'completed', but got '$STATUS'"
417 run_test 6b "LFSCK resumes from last checkpoint (2)"
422 echo "start $SINGLEMDS"
423 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
424 error "(1) Fail to start MDS!"
426 #define OBD_FAIL_LFSCK_DELAY2 0x1601
427 do_facet $SINGLEMDS $LCTL set_param fail_val=1
428 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
429 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
431 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
432 [ "$STATUS" == "scanning-phase1" ] ||
433 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
435 # Sleep 3 sec to guarantee at least one object processed by LFSCK
437 echo "stop $SINGLEMDS"
438 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
440 echo "start $SINGLEMDS"
441 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
442 error "(5) Fail to start MDS!"
444 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
445 [ "$STATUS" == "scanning-phase1" ] ||
446 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
448 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
449 do_facet $SINGLEMDS $LCTL set_param fail_val=0
451 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
452 [ "$STATUS" == "completed" ] ||
453 error "(7) Expect 'completed', but got '$STATUS'"
455 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
458 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
459 skip "Testing on UP system, the speed may be inaccurate."
464 echo "start $SINGLEMDS"
465 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
466 error "(1) Fail to start MDS!"
468 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "init" ] ||
470 error "(2) Expect 'init', but got '$STATUS'"
472 $START_NAMESPACE -s 100 || error "(3) Fail to start LFSCK!"
475 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
476 [ "$STATUS" == "scanning-phase1" ] ||
477 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
479 local SPEED=$($SHOW_NAMESPACE |
480 awk '/^average_speed_phase1/ { print $2 }')
481 # (100 * (10 + 1)) / 10 = 110
482 [ $SPEED -lt 120 ] ||
483 error "(4) Unexpected speed $SPEED, should not more than 120"
486 do_facet $SINGLEMDS \
487 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
490 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
491 # (100 * (10 - 1) + 300 * (10 - 1)) / 20 = 180
492 [ $SPEED -lt 170 ] &&
493 error "(5) Unexpected speed $SPEED, should not less than 170"
495 # (100 * (10 + 1) + 300 * (10 + 1)) / 20 = 220
496 [ $SPEED -lt 230 ] ||
497 error "(6) Unexpected speed $SPEED, should not more than 230"
499 do_facet $SINGLEMDS \
500 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
502 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
503 [ "$STATUS" == "completed" ] ||
504 error "(7) Expect 'completed', but got '$STATUS'"
506 run_test 9a "LFSCK speed control (1)"
508 $LCTL set_param debug=-lfsck > /dev/null || true
510 # restore MDS/OST size
511 MDSSIZE=${SAVED_MDSSIZE}
512 OSTSIZE=${SAVED_OSTSIZE}
514 # cleanup the system at last