3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != "ldiskfs" ] &&
21 skip "test LFSCK only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 MCREATE=${MCREATE:-mcreate}
25 SAVED_MDSSIZE=${MDSSIZE}
26 SAVED_OSTSIZE=${OSTSIZE}
27 # use small MDS + OST size to speed formatting time
28 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
32 check_and_setup_lustre
34 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
35 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
38 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
39 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
43 $LCTL set_param debug=+lfsck > /dev/null || true
45 MDT_DEV="${FSNAME}-MDT0000"
46 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
47 START_NAMESPACE="do_facet $SINGLEMDS \
48 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
49 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
50 SHOW_NAMESPACE="do_facet $SINGLEMDS \
51 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
52 MOUNT_OPTS_SCRUB="-o user_xattr"
53 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
66 if [ ! -z $igif ]; then
67 #define OBD_FAIL_FID_IGIF 0x1504
68 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
71 echo "preparing... ${nfiles} * ${ndirs} files will be created."
73 cp $LUSTRE/tests/*.sh $DIR/
74 for ((i = 0; i < ${ndirs}; i++)); do
75 mkdir $DIR/$tdir/d${i}
76 touch $DIR/$tdir/f${i}
77 for ((j = 0; j < ${nfiles}; j++)); do
78 touch $DIR/$tdir/d${i}/f${j}
80 mkdir $DIR/$tdir/e${i}
83 if [ ! -z $igif ]; then
84 touch $DIR/$tdir/dummy
85 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
89 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
90 echo "stop $SINGLEMDS"
91 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
96 echo "start $SINGLEMDS"
97 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
98 error "(1) Fail to start MDS!"
100 #define OBD_FAIL_LFSCK_DELAY1 0x1600
101 do_facet $SINGLEMDS $LCTL set_param fail_val=3
102 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
103 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
105 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
107 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
108 [ "$STATUS" == "scanning-phase1" ] ||
109 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
111 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
113 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
114 [ "$STATUS" == "stopped" ] ||
115 error "(6) Expect 'stopped', but got '$STATUS'"
117 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
119 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
120 [ "$STATUS" == "scanning-phase1" ] ||
121 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
123 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
124 do_facet $SINGLEMDS $LCTL set_param fail_val=0
126 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
127 [ "$STATUS" == "completed" ] ||
128 error "(9) Expect 'completed', but got '$STATUS'"
130 local repaired=$($SHOW_NAMESPACE |
131 awk '/^updated_phase1/ { print $2 }')
132 [ $repaired -eq 0 ] ||
133 error "(10) Expect nothing to be repaired, but got: $repaired"
135 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
136 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
139 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
140 [ "$STATUS" == "completed" ] ||
141 error "(12) Expect 'completed', but got '$STATUS'"
143 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
144 [ $((scanned1 + 1)) -eq $scanned2 ] ||
145 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
147 echo "stopall, should NOT crash LU-3649"
150 run_test 0 "Control LFSCK manually"
154 echo "start $SINGLEMDS"
155 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
156 error "(1) Fail to start MDS!"
158 mount_client $MOUNT || error "(2) Fail to start client!"
160 #define OBD_FAIL_FID_INDIR 0x1501
161 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
162 touch $DIR/$tdir/dummy
164 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
166 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
169 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
170 [ "$STATUS" == "completed" ] ||
171 error "(4) Expect 'completed', but got '$STATUS'"
173 local repaired=$($SHOW_NAMESPACE |
174 awk '/^updated_phase1/ { print $2 }')
175 [ $repaired -eq 1 ] ||
176 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
178 mount_client $MOUNT || error "(6) Fail to start client!"
180 #define OBD_FAIL_FID_LOOKUP 0x1505
181 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
182 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
184 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
186 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
191 echo "start $SINGLEMDS"
192 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
193 error "(1) Fail to start MDS!"
195 mount_client $MOUNT || error "(2) Fail to start client!"
197 #define OBD_FAIL_FID_INLMA 0x1502
198 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
199 touch $DIR/$tdir/dummy
201 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
203 #define OBD_FAIL_FID_NOLMA 0x1506
204 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
205 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
208 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
209 [ "$STATUS" == "completed" ] ||
210 error "(4) Expect 'completed', but got '$STATUS'"
212 local repaired=$($SHOW_NAMESPACE |
213 awk '/^updated_phase1/ { print $2 }')
214 [ $repaired -eq 1 ] ||
215 error "(5) Fail to repair missed FID-in-LMA: $repaired"
217 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
218 mount_client $MOUNT || error "(6) Fail to start client!"
220 #define OBD_FAIL_FID_LOOKUP 0x1505
221 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
222 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
224 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
226 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
230 echo "start $SINGLEMDS"
231 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
232 error "(1) Fail to start MDS!"
234 mount_client $MOUNT || error "(2) Fail to start client!"
236 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
237 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
238 touch $DIR/$tdir/dummy
240 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
242 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
245 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
246 [ "$STATUS" == "completed" ] ||
247 error "(4) Expect 'completed', but got '$STATUS'"
249 local repaired=$($SHOW_NAMESPACE |
250 awk '/^updated_phase1/ { print $2 }')
251 [ $repaired -eq 1 ] ||
252 error "(5) Fail to repair crashed linkEA: $repaired"
254 mount_client $MOUNT || error "(6) Fail to start client!"
256 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
257 error "(7) Fail to stat $DIR/$tdir/dummy"
259 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
260 local dummyname=$($LFS fid2path $DIR $dummyfid)
261 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
262 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
264 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
269 echo "start $SINGLEMDS"
270 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
271 error "(1) Fail to start MDS!"
273 mount_client $MOUNT || error "(2) Fail to start client!"
275 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
276 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
277 touch $DIR/$tdir/dummy
279 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
281 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
284 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
285 [ "$STATUS" == "completed" ] ||
286 error "(4) Expect 'completed', but got '$STATUS'"
288 local repaired=$($SHOW_NAMESPACE |
289 awk '/^updated_phase2/ { print $2 }')
290 [ $repaired -eq 1 ] ||
291 error "(5) Fail to repair crashed linkEA: $repaired"
293 mount_client $MOUNT || error "(6) Fail to start client!"
295 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
296 error "(7) Fail to stat $DIR/$tdir/dummy"
298 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
299 local dummyname=$($LFS fid2path $DIR $dummyfid)
300 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
301 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
303 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
308 echo "start $SINGLEMDS"
309 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
310 error "(1) Fail to start MDS!"
312 mount_client $MOUNT || error "(2) Fail to start client!"
314 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
315 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
316 touch $DIR/$tdir/dummy
318 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
320 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
323 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
324 [ "$STATUS" == "completed" ] ||
325 error "(4) Expect 'completed', but got '$STATUS'"
327 local repaired=$($SHOW_NAMESPACE |
328 awk '/^updated_phase2/ { print $2 }')
329 [ $repaired -eq 1 ] ||
330 error "(5) Fail to repair crashed linkEA: $repaired"
332 mount_client $MOUNT || error "(6) Fail to start client!"
334 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
335 error "(7) Fail to stat $DIR/$tdir/dummy"
337 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
338 local dummyname=$($LFS fid2path $DIR $dummyfid)
339 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
340 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
342 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
347 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
348 echo "start $SINGLEMDS with disabling OI scrub"
349 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
350 error "(2) Fail to start MDS!"
352 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
353 [ "$STATUS" == "init" ] ||
354 error "(3) Expect 'init', but got '$STATUS'"
356 #define OBD_FAIL_LFSCK_DELAY2 0x1601
357 do_facet $SINGLEMDS $LCTL set_param fail_val=1
358 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
359 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
362 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
363 [ "$STATUS" == "scanning-phase1" ] ||
364 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
366 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
367 [ "$FLAGS" == "inconsistent" ] ||
368 error "(6) Expect 'inconsistent', but got '$FLAGS'"
370 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
371 do_facet $SINGLEMDS $LCTL set_param fail_val=0
373 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
374 [ "$STATUS" == "completed" ] ||
375 error "(7) Expect 'completed', but got '$STATUS'"
377 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
378 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
380 local repaired=$($SHOW_NAMESPACE |
381 awk '/^updated_phase1/ { print $2 }')
382 [ $repaired -ge 9 ] ||
383 error "(9) Fail to repair crashed linkEA: $repaired"
385 mount_client $MOUNT || error "(10) Fail to start client!"
387 #define OBD_FAIL_FID_LOOKUP 0x1505
388 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
389 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
391 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
393 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
398 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
399 echo "start $SINGLEMDS with disabling OI scrub"
400 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
401 error "(2) Fail to start MDS!"
403 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
404 [ "$STATUS" == "init" ] ||
405 error "(3) Expect 'init', but got '$STATUS'"
407 #define OBD_FAIL_LFSCK_DELAY2 0x1601
408 do_facet $SINGLEMDS $LCTL set_param fail_val=1
409 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
410 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
413 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
414 [ "$STATUS" == "scanning-phase1" ] ||
415 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
417 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
418 [ "$FLAGS" == "inconsistent,upgrade" ] ||
419 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
421 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
422 do_facet $SINGLEMDS $LCTL set_param fail_val=0
424 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
425 [ "$STATUS" == "completed" ] ||
426 error "(7) Expect 'completed', but got '$STATUS'"
428 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
429 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
431 local repaired=$($SHOW_NAMESPACE |
432 awk '/^updated_phase1/ { print $2 }')
433 [ $repaired -ge 2 ] ||
434 error "(9) Fail to repair crashed linkEA: $repaired"
436 mount_client $MOUNT || error "(10) Fail to start client!"
438 #define OBD_FAIL_FID_LOOKUP 0x1505
439 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
440 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
442 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
444 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
445 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
446 local dummyname=$($LFS fid2path $DIR $dummyfid)
447 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
448 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
450 run_test 5 "LFSCK can handle IFIG object upgrading"
454 echo "start $SINGLEMDS"
455 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
456 error "(1) Fail to start MDS!"
458 #define OBD_FAIL_LFSCK_DELAY1 0x1600
459 do_facet $SINGLEMDS $LCTL set_param fail_val=1
460 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
461 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
463 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
464 [ "$STATUS" == "scanning-phase1" ] ||
465 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
467 # Sleep 3 sec to guarantee at least one object processed by LFSCK
469 # Fail the LFSCK to guarantee there is at least one checkpoint
470 #define OBD_FAIL_LFSCK_FATAL1 0x1608
471 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
473 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
474 [ "$STATUS" == "failed" ] ||
475 error "(4) Expect 'failed', but got '$STATUS'"
477 local POSITION0=$($SHOW_NAMESPACE |
478 awk '/^last_checkpoint_position/ { print $2 }' |
481 #define OBD_FAIL_LFSCK_DELAY1 0x1600
482 do_facet $SINGLEMDS $LCTL set_param fail_val=1
483 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
484 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
486 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
487 [ "$STATUS" == "scanning-phase1" ] ||
488 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
490 local POSITION1=$($SHOW_NAMESPACE |
491 awk '/^latest_start_position/ { print $2 }' |
493 [ $POSITION0 -lt $POSITION1 ] ||
494 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
496 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
497 do_facet $SINGLEMDS $LCTL set_param fail_val=0
499 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
500 [ "$STATUS" == "completed" ] ||
501 error "(8) Expect 'completed', but got '$STATUS'"
503 run_test 6a "LFSCK resumes from last checkpoint (1)"
507 echo "start $SINGLEMDS"
508 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
509 error "(1) Fail to start MDS!"
511 #define OBD_FAIL_LFSCK_DELAY2 0x1601
512 do_facet $SINGLEMDS $LCTL set_param fail_val=1
513 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
514 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
516 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
517 [ "$STATUS" == "scanning-phase1" ] ||
518 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
520 # Sleep 3 sec to guarantee at least one object processed by LFSCK
522 # Fail the LFSCK to guarantee there is at least one checkpoint
523 #define OBD_FAIL_LFSCK_FATAL2 0x1609
524 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
526 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
527 [ "$STATUS" == "failed" ] ||
528 error "(4) Expect 'failed', but got '$STATUS'"
530 local POSITION0=$($SHOW_NAMESPACE |
531 awk '/^last_checkpoint_position/ { print $4 }')
533 #define OBD_FAIL_LFSCK_DELAY2 0x1601
534 do_facet $SINGLEMDS $LCTL set_param fail_val=1
535 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
536 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
538 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
539 [ "$STATUS" == "scanning-phase1" ] ||
540 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
542 local POSITION1=$($SHOW_NAMESPACE |
543 awk '/^latest_start_position/ { print $4 }')
544 if [ $POSITION0 -gt $POSITION1 ]; then
545 [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
546 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
549 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
550 do_facet $SINGLEMDS $LCTL set_param fail_val=0
552 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
553 [ "$STATUS" == "completed" ] ||
554 error "(8) Expect 'completed', but got '$STATUS'"
556 run_test 6b "LFSCK resumes from last checkpoint (2)"
561 echo "start $SINGLEMDS"
562 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
563 error "(1) Fail to start MDS!"
565 #define OBD_FAIL_LFSCK_DELAY2 0x1601
566 do_facet $SINGLEMDS $LCTL set_param fail_val=1
567 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
568 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
570 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
571 [ "$STATUS" == "scanning-phase1" ] ||
572 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
574 # Sleep 3 sec to guarantee at least one object processed by LFSCK
576 echo "stop $SINGLEMDS"
577 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
579 echo "start $SINGLEMDS"
580 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
581 error "(5) Fail to start MDS!"
583 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
584 [ "$STATUS" == "scanning-phase1" ] ||
585 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
587 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
588 do_facet $SINGLEMDS $LCTL set_param fail_val=0
590 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
591 [ "$STATUS" == "completed" ] ||
592 error "(7) Expect 'completed', but got '$STATUS'"
594 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
599 echo "start $SINGLEMDS"
600 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
601 error "(1) Fail to start MDS!"
603 mount_client $MOUNT || error "(2) Fail to start client!"
605 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
606 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
607 for ((i = 0; i < 20; i++)); do
608 touch $DIR/$tdir/dummy${i}
611 #define OBD_FAIL_LFSCK_DELAY3 0x1602
612 do_facet $SINGLEMDS $LCTL set_param fail_val=1
613 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
614 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
617 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
618 [ "$STATUS" == "scanning-phase2" ] ||
619 error "(4) Expect 'scanning-phase2', but got '$STATUS'"
621 echo "stop $SINGLEMDS"
622 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
624 echo "start $SINGLEMDS"
625 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
626 error "(6) Fail to start MDS!"
628 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
629 [ "$STATUS" == "scanning-phase2" ] ||
630 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
632 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
633 do_facet $SINGLEMDS $LCTL set_param fail_val=0
635 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
636 [ "$STATUS" == "completed" ] ||
637 error "(8) Expect 'completed', but got '$STATUS'"
639 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
644 echo "start $SINGLEMDS"
645 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
646 error "(1) Fail to start MDS!"
648 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "init" ] ||
650 error "(2) Expect 'init', but got '$STATUS'"
652 mount_client $MOUNT || error "(3) Fail to start client!"
654 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
655 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
656 mkdir $DIR/$tdir/crashed
658 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
659 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
660 for ((i = 0; i < 5; i++)); do
661 touch $DIR/$tdir/dummy${i}
664 #define OBD_FAIL_LFSCK_DELAY2 0x1601
665 do_facet $SINGLEMDS $LCTL set_param fail_val=2
666 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
667 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
669 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
670 [ "$STATUS" == "scanning-phase1" ] ||
671 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
673 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
675 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
676 [ "$STATUS" == "stopped" ] ||
677 error "(7) Expect 'stopped', but got '$STATUS'"
679 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
681 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
682 [ "$STATUS" == "scanning-phase1" ] ||
683 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
685 #define OBD_FAIL_LFSCK_FATAL2 0x1609
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
688 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
689 [ "$STATUS" == "failed" ] ||
690 error "(10) Expect 'failed', but got '$STATUS'"
692 #define OBD_FAIL_LFSCK_DELAY1 0x1600
693 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
694 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
696 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
697 [ "$STATUS" == "scanning-phase1" ] ||
698 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
700 #define OBD_FAIL_LFSCK_CRASH 0x160a
701 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
704 echo "stop $SINGLEMDS"
705 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
707 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
708 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
710 echo "start $SINGLEMDS"
711 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
712 error "(14) Fail to start MDS!"
714 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
715 [ "$STATUS" == "crashed" ] ||
716 error "(15) Expect 'crashed', but got '$STATUS'"
718 #define OBD_FAIL_LFSCK_DELAY2 0x1601
719 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
720 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
722 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
723 [ "$STATUS" == "scanning-phase1" ] ||
724 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
726 echo "stop $SINGLEMDS"
727 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
729 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
730 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
732 echo "start $SINGLEMDS"
733 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
734 error "(19) Fail to start MDS!"
736 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
737 [ "$STATUS" == "paused" ] ||
738 error "(20) Expect 'paused', but got '$STATUS'"
740 #define OBD_FAIL_LFSCK_DELAY3 0x1602
741 do_facet $SINGLEMDS $LCTL set_param fail_val=2
742 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
744 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
746 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
747 [ "$STATUS" == "scanning-phase2" ] ||
748 error "(22) Expect 'scanning-phase2', but got '$STATUS'"
750 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
751 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
752 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
754 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
755 do_facet $SINGLEMDS $LCTL set_param fail_val=0
757 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
758 [ "$STATUS" == "completed" ] ||
759 error "(24) Expect 'completed', but got '$STATUS'"
761 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
762 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
765 run_test 8 "LFSCK state machine"
768 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
769 skip "Testing on UP system, the speed may be inaccurate."
774 echo "start $SINGLEMDS"
775 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
776 error "(1) Fail to start MDS!"
778 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
779 [ "$STATUS" == "init" ] ||
780 error "(2) Expect 'init', but got '$STATUS'"
782 local BASE_SPEED1=100
784 $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
787 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
788 [ "$STATUS" == "scanning-phase1" ] ||
789 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
791 local SPEED=$($SHOW_NAMESPACE |
792 awk '/^average_speed_phase1/ { print $2 }')
794 # There may be time error, normally it should be less than 2 seconds.
795 # We allow another 20% schedule error.
797 # MAX_MARGIN = 1.2 = 12 / 10
798 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
799 RUN_TIME1 * 12 / 10))
800 [ $SPEED -lt $MAX_SPEED ] ||
801 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
804 local BASE_SPEED2=300
806 do_facet $SINGLEMDS \
807 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
810 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
811 # MIN_MARGIN = 0.8 = 8 / 10
812 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
813 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
814 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
815 [ $SPEED -gt $MIN_SPEED ] ||
816 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
818 # MAX_MARGIN = 1.2 = 12 / 10
819 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
820 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
821 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
822 [ $SPEED -lt $MAX_SPEED ] ||
823 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
825 do_facet $SINGLEMDS \
826 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
828 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
829 [ "$STATUS" == "completed" ] ||
830 error "(7) Expect 'completed', but got '$STATUS'"
832 run_test 9a "LFSCK speed control (1)"
835 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
836 skip "Testing on UP system, the speed may be inaccurate."
841 echo "start $SINGLEMDS"
842 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
843 error "(1) Fail to start MDS!"
845 mount_client $MOUNT || error "(2) Fail to start client!"
847 echo "Another preparing... 50 * 50 files (with error) will be created."
848 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
849 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
850 for ((i = 0; i < 50; i++)); do
851 mkdir -p $DIR/$tdir/d${i}
852 touch $DIR/$tdir/f${i}
853 for ((j = 0; j < 50; j++)); do
854 touch $DIR/$tdir/d${i}/f${j}
858 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
859 [ "$STATUS" == "init" ] ||
860 error "(3) Expect 'init', but got '$STATUS'"
862 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
863 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
864 $START_NAMESPACE || error "(4) Fail to start LFSCK!"
867 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
868 [ "$STATUS" == "stopped" ] ||
869 error "(5) Expect 'stopped', but got '$STATUS'"
871 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
875 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
878 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
879 [ "$STATUS" == "scanning-phase2" ] ||
880 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
882 local SPEED=$($SHOW_NAMESPACE |
883 awk '/^average_speed_phase2/ { print $2 }')
884 # There may be time error, normally it should be less than 2 seconds.
885 # We allow another 20% schedule error.
887 # MAX_MARGIN = 1.2 = 12 / 10
888 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
889 RUN_TIME1 * 12 / 10))
890 [ $SPEED -lt $MAX_SPEED ] ||
891 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
894 local BASE_SPEED2=150
896 do_facet $SINGLEMDS \
897 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
900 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
901 # MIN_MARGIN = 0.8 = 8 / 10
902 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
903 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
904 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
905 [ $SPEED -gt $MIN_SPEED ] ||
906 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
908 # MAX_MARGIN = 1.2 = 12 / 10
909 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
910 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
911 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
912 [ $SPEED -lt $MAX_SPEED ] ||
913 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
915 do_facet $SINGLEMDS \
916 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
918 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
919 [ "$STATUS" == "completed" ] ||
920 error "(11) Expect 'completed', but got '$STATUS'"
922 run_test 9b "LFSCK speed control (2)"
927 echo "start $SINGLEMDS"
928 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
929 error "(1) Fail to start MDS!"
931 mount_client $MOUNT || error "(2) Fail to start client!"
933 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
934 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
935 for ((i = 0; i < 1000; i = $((i+2)))); do
936 mkdir -p $DIR/$tdir/d${i}
937 touch $DIR/$tdir/f${i}
938 for ((j = 0; j < 5; j++)); do
939 touch $DIR/$tdir/d${i}/f${j}
943 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
944 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
945 for ((i = 1; i < 1000; i = $((i+2)))); do
946 mkdir -p $DIR/$tdir/d${i}
947 touch $DIR/$tdir/f${i}
948 for ((j = 0; j < 5; j++)); do
949 touch $DIR/$tdir/d${i}/f${j}
953 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
954 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
957 mount_client $MOUNT || error "(3) Fail to start client!"
959 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
960 [ "$STATUS" == "init" ] ||
961 error "(4) Expect 'init', but got '$STATUS'"
963 $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
966 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
967 [ "$STATUS" == "scanning-phase1" ] ||
968 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
970 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
972 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
974 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
976 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
978 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
980 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
982 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
984 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
985 error "(14) Fail to softlink!"
987 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
988 [ "$STATUS" == "scanning-phase1" ] ||
989 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
991 do_facet $SINGLEMDS \
992 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
995 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
996 [ "$STATUS" == "completed" ] ||
997 error "(16) Expect 'completed', but got '$STATUS'"
999 run_test 10 "System is available during LFSCK scanning"
1001 $LCTL set_param debug=-lfsck > /dev/null || true
1003 # restore MDS/OST size
1004 MDSSIZE=${SAVED_MDSSIZE}
1005 OSTSIZE=${SAVED_OSTSIZE}
1007 # cleanup the system at last