3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
21 skip "test LFSCK only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 MCREATE=${MCREATE:-mcreate}
25 SAVED_MDSSIZE=${MDSSIZE}
26 SAVED_OSTSIZE=${OSTSIZE}
27 # use small MDS + OST size to speed formatting time
28 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
32 check_and_setup_lustre
34 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
35 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
40 $LCTL set_param debug=+lfsck > /dev/null || true
42 MDT_DEV="${FSNAME}-MDT0000"
43 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
44 START_NAMESPACE="do_facet $SINGLEMDS \
45 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
46 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
47 SHOW_NAMESPACE="do_facet $SINGLEMDS \
48 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
49 MOUNT_OPTS_SCRUB="-o user_xattr"
50 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
63 if [ ! -z $igif ]; then
64 #define OBD_FAIL_FID_IGIF 0x1504
65 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
68 echo "preparing... ${nfiles} * ${ndirs} files will be created."
70 cp $LUSTRE/tests/*.sh $DIR/
71 for ((i = 0; i < ${ndirs}; i++)); do
72 mkdir $DIR/$tdir/d${i}
73 touch $DIR/$tdir/f${i}
74 for ((j = 0; j < ${nfiles}; j++)); do
75 touch $DIR/$tdir/d${i}/f${j}
77 mkdir $DIR/$tdir/e${i}
80 if [ ! -z $igif ]; then
81 touch $DIR/$tdir/dummy
82 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
86 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
87 echo "stop $SINGLEMDS"
88 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
93 echo "start $SINGLEMDS"
94 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
95 error "(1) Fail to start MDS!"
97 #define OBD_FAIL_LFSCK_DELAY1 0x1600
98 do_facet $SINGLEMDS $LCTL set_param fail_val=3
99 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
100 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
102 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
104 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
105 [ "$STATUS" == "scanning-phase1" ] ||
106 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
108 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
110 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
111 [ "$STATUS" == "stopped" ] ||
112 error "(6) Expect 'stopped', but got '$STATUS'"
114 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
116 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
117 [ "$STATUS" == "scanning-phase1" ] ||
118 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
120 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
121 do_facet $SINGLEMDS $LCTL set_param fail_val=0
123 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
124 [ "$STATUS" == "completed" ] ||
125 error "(9) Expect 'completed', but got '$STATUS'"
127 local repaired=$($SHOW_NAMESPACE |
128 awk '/^updated_phase1/ { print $2 }')
129 [ $repaired -eq 0 ] ||
130 error "(10) Expect nothing to be repaired, but got: $repaired"
132 run_test 0 "Control LFSCK manually"
136 echo "start $SINGLEMDS"
137 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
138 error "(1) Fail to start MDS!"
140 mount_client $MOUNT || error "(2) Fail to start client!"
142 #define OBD_FAIL_FID_INDIR 0x1501
143 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
144 touch $DIR/$tdir/dummy
146 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
148 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
151 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
152 [ "$STATUS" == "completed" ] ||
153 error "(4) Expect 'completed', but got '$STATUS'"
155 local repaired=$($SHOW_NAMESPACE |
156 awk '/^updated_phase1/ { print $2 }')
157 [ $repaired -eq 1 ] ||
158 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
160 mount_client $MOUNT || error "(6) Fail to start client!"
162 #define OBD_FAIL_FID_LOOKUP 0x1505
163 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
164 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
166 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
168 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
173 echo "start $SINGLEMDS"
174 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
175 error "(1) Fail to start MDS!"
177 mount_client $MOUNT || error "(2) Fail to start client!"
179 #define OBD_FAIL_FID_INLMA 0x1502
180 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
181 touch $DIR/$tdir/dummy
183 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
185 #define OBD_FAIL_FID_NOLMA 0x1506
186 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
187 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
190 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
191 [ "$STATUS" == "completed" ] ||
192 error "(4) Expect 'completed', but got '$STATUS'"
194 local repaired=$($SHOW_NAMESPACE |
195 awk '/^updated_phase1/ { print $2 }')
196 [ $repaired -eq 1 ] ||
197 error "(5) Fail to repair missed FID-in-LMA: $repaired"
199 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
200 mount_client $MOUNT || error "(6) Fail to start client!"
202 #define OBD_FAIL_FID_LOOKUP 0x1505
203 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
204 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
206 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
208 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
212 echo "start $SINGLEMDS"
213 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
214 error "(1) Fail to start MDS!"
216 mount_client $MOUNT || error "(2) Fail to start client!"
218 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
219 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
220 touch $DIR/$tdir/dummy
222 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
224 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
227 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
228 [ "$STATUS" == "completed" ] ||
229 error "(4) Expect 'completed', but got '$STATUS'"
231 local repaired=$($SHOW_NAMESPACE |
232 awk '/^updated_phase1/ { print $2 }')
233 [ $repaired -eq 1 ] ||
234 error "(5) Fail to repair crashed linkEA: $repaired"
236 mount_client $MOUNT || error "(6) Fail to start client!"
238 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
239 error "(7) Fail to stat $DIR/$tdir/dummy"
241 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
242 local dummyname=$($LFS fid2path $DIR $dummyfid)
243 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
244 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
246 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
251 echo "start $SINGLEMDS"
252 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
253 error "(1) Fail to start MDS!"
255 mount_client $MOUNT || error "(2) Fail to start client!"
257 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
258 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
259 touch $DIR/$tdir/dummy
261 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
263 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
266 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
267 [ "$STATUS" == "completed" ] ||
268 error "(4) Expect 'completed', but got '$STATUS'"
270 local repaired=$($SHOW_NAMESPACE |
271 awk '/^updated_phase2/ { print $2 }')
272 [ $repaired -eq 1 ] ||
273 error "(5) Fail to repair crashed linkEA: $repaired"
275 mount_client $MOUNT || error "(6) Fail to start client!"
277 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
278 error "(7) Fail to stat $DIR/$tdir/dummy"
280 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
281 local dummyname=$($LFS fid2path $DIR $dummyfid)
282 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
283 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
285 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
290 echo "start $SINGLEMDS"
291 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
292 error "(1) Fail to start MDS!"
294 mount_client $MOUNT || error "(2) Fail to start client!"
296 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
297 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
298 touch $DIR/$tdir/dummy
300 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
302 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
305 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
306 [ "$STATUS" == "completed" ] ||
307 error "(4) Expect 'completed', but got '$STATUS'"
309 local repaired=$($SHOW_NAMESPACE |
310 awk '/^updated_phase2/ { print $2 }')
311 [ $repaired -eq 1 ] ||
312 error "(5) Fail to repair crashed linkEA: $repaired"
314 mount_client $MOUNT || error "(6) Fail to start client!"
316 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
317 error "(7) Fail to stat $DIR/$tdir/dummy"
319 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
320 local dummyname=$($LFS fid2path $DIR $dummyfid)
321 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
322 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
324 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
329 mds_backup_restore || error "(1) Fail to backup/restore!"
330 echo "start $SINGLEMDS with disabling OI scrub"
331 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
332 error "(2) Fail to start MDS!"
334 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
335 [ "$STATUS" == "init" ] ||
336 error "(3) Expect 'init', but got '$STATUS'"
338 #define OBD_FAIL_LFSCK_DELAY2 0x1601
339 do_facet $SINGLEMDS $LCTL set_param fail_val=1
340 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
341 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
344 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
345 [ "$STATUS" == "scanning-phase1" ] ||
346 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
348 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
349 [ "$FLAGS" == "inconsistent" ] ||
350 error "(6) Expect 'inconsistent', but got '$FLAGS'"
352 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
353 do_facet $SINGLEMDS $LCTL set_param fail_val=0
355 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
356 [ "$STATUS" == "completed" ] ||
357 error "(7) Expect 'completed', but got '$STATUS'"
359 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
360 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
362 local repaired=$($SHOW_NAMESPACE |
363 awk '/^updated_phase1/ { print $2 }')
364 [ $repaired -ge 9 ] ||
365 error "(9) Fail to repair crashed linkEA: $repaired"
367 mount_client $MOUNT || error "(10) Fail to start client!"
369 #define OBD_FAIL_FID_LOOKUP 0x1505
370 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
371 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
373 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
375 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
380 mds_backup_restore 1 || error "(1) Fail to backup/restore!"
381 echo "start $SINGLEMDS with disabling OI scrub"
382 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
383 error "(2) Fail to start MDS!"
385 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
386 [ "$STATUS" == "init" ] ||
387 error "(3) Expect 'init', but got '$STATUS'"
389 #define OBD_FAIL_LFSCK_DELAY2 0x1601
390 do_facet $SINGLEMDS $LCTL set_param fail_val=1
391 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
392 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
395 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
396 [ "$STATUS" == "scanning-phase1" ] ||
397 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
399 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
400 [ "$FLAGS" == "inconsistent,upgrade" ] ||
401 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
403 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
404 do_facet $SINGLEMDS $LCTL set_param fail_val=0
406 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
407 [ "$STATUS" == "completed" ] ||
408 error "(7) Expect 'completed', but got '$STATUS'"
410 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
411 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
413 local repaired=$($SHOW_NAMESPACE |
414 awk '/^updated_phase1/ { print $2 }')
415 [ $repaired -ge 2 ] ||
416 error "(9) Fail to repair crashed linkEA: $repaired"
418 mount_client $MOUNT || error "(10) Fail to start client!"
420 #define OBD_FAIL_FID_LOOKUP 0x1505
421 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
422 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
424 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
426 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
427 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
428 local dummyname=$($LFS fid2path $DIR $dummyfid)
429 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
430 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
432 run_test 5 "LFSCK can handle IFIG object upgrading"
436 echo "start $SINGLEMDS"
437 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
438 error "(1) Fail to start MDS!"
440 #define OBD_FAIL_LFSCK_DELAY1 0x1600
441 do_facet $SINGLEMDS $LCTL set_param fail_val=1
442 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
443 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
445 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
446 [ "$STATUS" == "scanning-phase1" ] ||
447 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
449 # Sleep 3 sec to guarantee at least one object processed by LFSCK
451 # Fail the LFSCK to guarantee there is at least one checkpoint
452 #define OBD_FAIL_LFSCK_FATAL1 0x1608
453 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
455 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
456 [ "$STATUS" == "failed" ] ||
457 error "(4) Expect 'failed', but got '$STATUS'"
459 local POSITION0=$($SHOW_NAMESPACE |
460 awk '/^last_checkpoint_position/ { print $2 }' |
463 #define OBD_FAIL_LFSCK_DELAY1 0x1600
464 do_facet $SINGLEMDS $LCTL set_param fail_val=1
465 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
466 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
468 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "scanning-phase1" ] ||
470 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
472 local POSITION1=$($SHOW_NAMESPACE |
473 awk '/^latest_start_position/ { print $2 }' |
475 [ $POSITION0 -lt $POSITION1 ] ||
476 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
478 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
479 do_facet $SINGLEMDS $LCTL set_param fail_val=0
481 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
482 [ "$STATUS" == "completed" ] ||
483 error "(8) Expect 'completed', but got '$STATUS'"
485 run_test 6a "LFSCK resumes from last checkpoint (1)"
489 echo "start $SINGLEMDS"
490 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
491 error "(1) Fail to start MDS!"
493 #define OBD_FAIL_LFSCK_DELAY2 0x1601
494 do_facet $SINGLEMDS $LCTL set_param fail_val=1
495 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
496 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
498 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
499 [ "$STATUS" == "scanning-phase1" ] ||
500 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
502 # Sleep 3 sec to guarantee at least one object processed by LFSCK
504 # Fail the LFSCK to guarantee there is at least one checkpoint
505 #define OBD_FAIL_LFSCK_FATAL2 0x1609
506 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
508 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
509 [ "$STATUS" == "failed" ] ||
510 error "(4) Expect 'failed', but got '$STATUS'"
512 local POSITION0=$($SHOW_NAMESPACE |
513 awk '/^last_checkpoint_position/ { print $4 }')
515 #define OBD_FAIL_LFSCK_DELAY2 0x1601
516 do_facet $SINGLEMDS $LCTL set_param fail_val=1
517 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
518 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
520 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
521 [ "$STATUS" == "scanning-phase1" ] ||
522 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
524 local POSITION1=$($SHOW_NAMESPACE |
525 awk '/^latest_start_position/ { print $4 }')
526 if [ $POSITION0 -gt $POSITION1 ]; then
527 [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
528 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
531 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
532 do_facet $SINGLEMDS $LCTL set_param fail_val=0
534 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
535 [ "$STATUS" == "completed" ] ||
536 error "(8) Expect 'completed', but got '$STATUS'"
538 run_test 6b "LFSCK resumes from last checkpoint (2)"
543 echo "start $SINGLEMDS"
544 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
545 error "(1) Fail to start MDS!"
547 #define OBD_FAIL_LFSCK_DELAY2 0x1601
548 do_facet $SINGLEMDS $LCTL set_param fail_val=1
549 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
550 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
552 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
553 [ "$STATUS" == "scanning-phase1" ] ||
554 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
556 # Sleep 3 sec to guarantee at least one object processed by LFSCK
558 echo "stop $SINGLEMDS"
559 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
561 echo "start $SINGLEMDS"
562 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
563 error "(5) Fail to start MDS!"
565 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
566 [ "$STATUS" == "scanning-phase1" ] ||
567 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
569 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
570 do_facet $SINGLEMDS $LCTL set_param fail_val=0
572 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "completed" ] ||
574 error "(7) Expect 'completed', but got '$STATUS'"
576 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(1) Fail to start MDS!"
585 mount_client $MOUNT || error "(2) Fail to start client!"
587 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
588 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
589 for ((i = 0; i < 20; i++)); do
590 touch $DIR/$tdir/dummy${i}
593 #define OBD_FAIL_LFSCK_DELAY3 0x1602
594 do_facet $SINGLEMDS $LCTL set_param fail_val=1
595 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
596 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
599 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
600 [ "$STATUS" == "scanning-phase2" ] ||
601 error "(4) Expect 'scanning-phase2', but got '$STATUS'"
603 echo "stop $SINGLEMDS"
604 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
606 echo "start $SINGLEMDS"
607 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
608 error "(6) Fail to start MDS!"
610 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
611 [ "$STATUS" == "scanning-phase2" ] ||
612 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
614 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
615 do_facet $SINGLEMDS $LCTL set_param fail_val=0
617 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
618 [ "$STATUS" == "completed" ] ||
619 error "(8) Expect 'completed', but got '$STATUS'"
621 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
626 echo "start $SINGLEMDS"
627 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
628 error "(1) Fail to start MDS!"
630 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
631 [ "$STATUS" == "init" ] ||
632 error "(2) Expect 'init', but got '$STATUS'"
634 mount_client $MOUNT || error "(3) Fail to start client!"
636 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
637 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
638 mkdir $DIR/$tdir/crashed
640 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
641 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
642 for ((i = 0; i < 5; i++)); do
643 touch $DIR/$tdir/dummy${i}
646 #define OBD_FAIL_LFSCK_DELAY2 0x1601
647 do_facet $SINGLEMDS $LCTL set_param fail_val=2
648 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
649 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
651 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
652 [ "$STATUS" == "scanning-phase1" ] ||
653 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
655 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
657 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
658 [ "$STATUS" == "stopped" ] ||
659 error "(7) Expect 'stopped', but got '$STATUS'"
661 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
663 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
664 [ "$STATUS" == "scanning-phase1" ] ||
665 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
667 #define OBD_FAIL_LFSCK_FATAL2 0x1609
668 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
670 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
671 [ "$STATUS" == "failed" ] ||
672 error "(10) Expect 'failed', but got '$STATUS'"
674 #define OBD_FAIL_LFSCK_DELAY1 0x1600
675 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
676 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
678 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
679 [ "$STATUS" == "scanning-phase1" ] ||
680 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
682 #define OBD_FAIL_LFSCK_CRASH 0x160a
683 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
686 echo "stop $SINGLEMDS"
687 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
689 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
690 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
692 echo "start $SINGLEMDS"
693 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
694 error "(14) Fail to start MDS!"
696 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
697 [ "$STATUS" == "crashed" ] ||
698 error "(15) Expect 'crashed', but got '$STATUS'"
700 #define OBD_FAIL_LFSCK_DELAY2 0x1601
701 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
702 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
704 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
705 [ "$STATUS" == "scanning-phase1" ] ||
706 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
708 echo "stop $SINGLEMDS"
709 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
711 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
712 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
714 echo "start $SINGLEMDS"
715 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
716 error "(19) Fail to start MDS!"
718 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
719 [ "$STATUS" == "paused" ] ||
720 error "(20) Expect 'paused', but got '$STATUS'"
722 #define OBD_FAIL_LFSCK_DELAY3 0x1602
723 do_facet $SINGLEMDS $LCTL set_param fail_val=2
724 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
726 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
728 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
729 [ "$STATUS" == "scanning-phase2" ] ||
730 error "(22) Expect 'scanning-phase2', but got '$STATUS'"
732 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
733 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
734 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
736 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
737 do_facet $SINGLEMDS $LCTL set_param fail_val=0
739 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
740 [ "$STATUS" == "completed" ] ||
741 error "(24) Expect 'completed', but got '$STATUS'"
743 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
744 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
747 run_test 8 "LFSCK state machine"
750 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
751 skip "Testing on UP system, the speed may be inaccurate."
756 echo "start $SINGLEMDS"
757 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
758 error "(1) Fail to start MDS!"
760 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
761 [ "$STATUS" == "init" ] ||
762 error "(2) Expect 'init', but got '$STATUS'"
764 local BASE_SPEED1=100
766 $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
769 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
770 [ "$STATUS" == "scanning-phase1" ] ||
771 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
773 local SPEED=$($SHOW_NAMESPACE |
774 awk '/^average_speed_phase1/ { print $2 }')
776 # There may be time error, normally it should be less than 2 seconds.
777 # We allow another 20% schedule error.
779 # MAX_MARGIN = 1.2 = 12 / 10
780 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
781 RUN_TIME1 * 12 / 10))
782 [ $SPEED -lt $MAX_SPEED ] ||
783 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
786 local BASE_SPEED2=300
788 do_facet $SINGLEMDS \
789 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
792 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
793 # MIN_MARGIN = 0.8 = 8 / 10
794 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
795 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
796 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
797 [ $SPEED -gt $MIN_SPEED ] ||
798 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
800 # MAX_MARGIN = 1.2 = 12 / 10
801 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
802 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
803 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
804 [ $SPEED -lt $MAX_SPEED ] ||
805 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
807 do_facet $SINGLEMDS \
808 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
810 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
811 [ "$STATUS" == "completed" ] ||
812 error "(7) Expect 'completed', but got '$STATUS'"
814 run_test 9a "LFSCK speed control (1)"
817 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
818 skip "Testing on UP system, the speed may be inaccurate."
823 echo "start $SINGLEMDS"
824 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
825 error "(1) Fail to start MDS!"
827 mount_client $MOUNT || error "(2) Fail to start client!"
829 echo "Another preparing... 50 * 50 files (with error) will be created."
830 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
831 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
832 for ((i = 0; i < 50; i++)); do
833 mkdir -p $DIR/$tdir/d${i}
834 touch $DIR/$tdir/f${i}
835 for ((j = 0; j < 50; j++)); do
836 touch $DIR/$tdir/d${i}/f${j}
840 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
841 [ "$STATUS" == "init" ] ||
842 error "(3) Expect 'init', but got '$STATUS'"
844 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
845 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
846 $START_NAMESPACE || error "(4) Fail to start LFSCK!"
849 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
850 [ "$STATUS" == "stopped" ] ||
851 error "(5) Expect 'stopped', but got '$STATUS'"
853 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
857 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
860 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
861 [ "$STATUS" == "scanning-phase2" ] ||
862 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
864 local SPEED=$($SHOW_NAMESPACE |
865 awk '/^average_speed_phase2/ { print $2 }')
866 # There may be time error, normally it should be less than 2 seconds.
867 # We allow another 20% schedule error.
869 # MAX_MARGIN = 1.2 = 12 / 10
870 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
871 RUN_TIME1 * 12 / 10))
872 [ $SPEED -lt $MAX_SPEED ] ||
873 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
876 local BASE_SPEED2=150
878 do_facet $SINGLEMDS \
879 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
882 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
883 # MIN_MARGIN = 0.8 = 8 / 10
884 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
885 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
886 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
887 [ $SPEED -gt $MIN_SPEED ] ||
888 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
890 # MAX_MARGIN = 1.2 = 12 / 10
891 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
892 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
893 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
894 [ $SPEED -lt $MAX_SPEED ] ||
895 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
897 do_facet $SINGLEMDS \
898 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
900 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
901 [ "$STATUS" == "completed" ] ||
902 error "(11) Expect 'completed', but got '$STATUS'"
904 run_test 9b "LFSCK speed control (2)"
909 echo "start $SINGLEMDS"
910 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
911 error "(1) Fail to start MDS!"
913 mount_client $MOUNT || error "(2) Fail to start client!"
915 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
916 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
917 for ((i = 0; i < 1000; i = $((i+2)))); do
918 mkdir -p $DIR/$tdir/d${i}
919 touch $DIR/$tdir/f${i}
920 for ((j = 0; j < 5; j++)); do
921 touch $DIR/$tdir/d${i}/f${j}
925 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
926 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
927 for ((i = 1; i < 1000; i = $((i+2)))); do
928 mkdir -p $DIR/$tdir/d${i}
929 touch $DIR/$tdir/f${i}
930 for ((j = 0; j < 5; j++)); do
931 touch $DIR/$tdir/d${i}/f${j}
935 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
936 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
939 mount_client $MOUNT || error "(3) Fail to start client!"
941 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
942 [ "$STATUS" == "init" ] ||
943 error "(4) Expect 'init', but got '$STATUS'"
945 $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
948 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
949 [ "$STATUS" == "scanning-phase1" ] ||
950 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
952 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
954 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
956 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
958 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
960 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
962 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
964 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
966 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
967 error "(14) Fail to softlink!"
969 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
970 [ "$STATUS" == "scanning-phase1" ] ||
971 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
973 do_facet $SINGLEMDS \
974 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
977 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
978 [ "$STATUS" == "completed" ] ||
979 error "(16) Expect 'completed', but got '$STATUS'"
981 run_test 10 "System is available during LFSCK scanning"
983 $LCTL set_param debug=-lfsck > /dev/null || true
985 # restore MDS/OST size
986 MDSSIZE=${SAVED_MDSSIZE}
987 OSTSIZE=${SAVED_OSTSIZE}
989 # cleanup the system at last