3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 SAVED_OSTCOUNT=${OSTCOUNT}
25 # use small MDS + OST size to speed formatting time
26 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
29 # no need too much OSTs, to reduce the format/start/stop overhead
30 [ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
34 # build up a clean test environment.
38 [ $(facet_fstype $SINGLEMDS) != "ldiskfs" ] &&
39 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
41 [ $(facet_fstype ost1) != "ldiskfs" ] &&
42 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
44 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
45 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
48 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
49 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
51 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.1) ]] &&
52 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 15"
54 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.4.90) ]] &&
55 [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.4.50) ]] &&
56 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 15"
58 [[ $(lustre_version_code ost1) -lt $(version_code 2.4.50) ]] &&
59 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14"
61 [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.5.59) ]] &&
66 MDT_DEV="${FSNAME}-MDT0000"
67 OST_DEV="${FSNAME}-OST0000"
68 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
74 # use "lfsck_start -A" when we no longer need testing interop
75 for n in $(seq $MDSCOUNT); do
76 do_facet mds$n $LCTL lfsck_start -M $(facet_svc mds$n) \
78 error "($error_id) Failed to start OI scrub on mds$n"
86 # use "lfsck_stop -A" when we no longer need testing interop
87 for n in $(seq $MDSCOUNT); do
88 do_facet mds$n $LCTL lfsck_stop -M $(facet_svc mds$n) ||
89 error "($error_id) Failed to stop OI scrub on mds$n"
96 do_facet mds$n $LCTL get_param -n \
97 osd-ldiskfs.$(facet_svc mds$n).oi_scrub
100 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV} $SCRUB_ONLY"
101 START_SCRUB_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} $SCRUB_ONLY"
102 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
103 SHOW_SCRUB="do_facet $SINGLEMDS \
104 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
105 SHOW_SCRUB_ON_OST="do_facet ost1 \
106 $LCTL get_param -n osd-ldiskfs.${OST_DEV}.oi_scrub"
107 MOUNT_OPTS_SCRUB="-o user_xattr"
108 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
116 echo "preparing... $(date)"
117 for n in $(seq $MDSCOUNT); do
118 echo "creating $nfiles files on mds$n"
119 if [ $n -eq 1 ]; then
120 mkdir $DIR/$tdir/mds$n ||
121 error "Failed to create directory mds$n"
123 $LFS mkdir -i $((n - 1)) $DIR/$tdir/mds$n ||
124 error "Failed to create remote directory mds$n"
126 cp $LUSTRE/tests/*.sh $DIR/$tdir/mds$n ||
127 error "Failed to copy files to mds$n"
128 if [[ $nfiles -gt 0 ]]; then
129 createmany -m $DIR/$tdir/mds$n/$tfile $nfiles > \
130 /dev/null || error "createmany failed on mds$n"
133 echo "prepared $(date)."
134 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
135 for n in $(seq $MDSCOUNT); do
137 stop mds$n > /dev/null || error "Fail to stop MDS$n!"
146 for n in $(seq $MDSCOUNT); do
147 start mds$n $(mdsdevname $n) $opts >/dev/null ||
148 error "($error_id) Failed to start mds$n"
156 for n in $(seq $MDSCOUNT); do
157 echo "stopping mds$n"
158 stop mds$n >/dev/null ||
159 error "($error_id) Failed to stop mds$n"
163 scrub_check_status() {
168 for n in $(seq $MDSCOUNT); do
169 wait_update_facet mds$n "$LCTL get_param -n \
170 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
171 awk '/^status/ { print \\\$2 }'" "$expected" 6 ||
172 error "($error_id) Expected '$expected' on mds$n"
176 scrub_check_flags() {
182 for n in $(seq $MDSCOUNT); do
183 actual=$(do_facet mds$n $LCTL get_param -n \
184 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
185 awk '/^flags/ { print $2 }')
186 if [ "$actual" != "$expected" ]; then
187 error "($error_id) Expected '$expected' on mds$n, but" \
193 scrub_check_params() {
199 for n in $(seq $MDSCOUNT); do
200 actual=$(do_facet mds$n $LCTL get_param -n \
201 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
202 awk '/^param/ { print $2 }')
203 if [ "$actual" != "$expected" ]; then
204 error "($error_id) Expected '$expected' on mds$n, but" \
210 scrub_check_repaired() {
216 for n in $(seq $MDSCOUNT); do
217 actual=$(do_facet mds$n $LCTL get_param -n \
218 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
219 awk '/^updated/ { print $2 }')
221 if [ $expected -eq 0 -a $actual -ne 0 ]; then
222 error "($error_id) Expected no repaired on mds$n, but" \
226 if [ $expected -ne 0 -a $actual -lt $expected ]; then
227 error "($error_id) Expected '$expected' on mds$n, but" \
237 for n in $(seq $MDSCOUNT); do
238 diff -q $LUSTRE/tests/test-framework.sh \
239 $DIR/$tdir/mds$n/test-framework.sh ||
240 error "($error_id) File data check failed"
249 for n in $(seq $MDSCOUNT); do
250 mds_remove_ois mds$n $index ||
251 error "($error_id) Failed to remove OI .$index on mds$n"
255 scrub_backup_restore() {
260 for n in $(seq $MDSCOUNT); do
261 mds_backup_restore mds$n $igif ||
262 error "(error_id) Backup/restore on mds$n failed"
266 scrub_enable_auto() {
269 for n in $(seq $MDSCOUNT); do
270 do_facet mds$n $LCTL set_param -n \
271 osd-ldiskfs.$(facet_svc mds$n).auto_scrub 1
277 echo "starting MDTs without disabling OI scrub"
278 scrub_start_mds 1 "$MOUNT_OPTS_SCRUB"
279 scrub_check_status 2 init
280 scrub_check_flags 3 ""
281 mount_client $MOUNT || error "(4) Fail to start client!"
284 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
288 echo "start $SINGLEMDS without disabling OI scrub"
289 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
290 error "(1) Fail to start MDS!"
292 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
293 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
295 mount_client $MOUNT || error "(4) Fail to start client!"
296 #define OBD_FAIL_OSD_FID_MAPPING 0x193
297 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
298 # update .lustre OI mapping
300 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
301 umount_client $MOUNT || error "(5) Fail to stop client!"
303 echo "stop $SINGLEMDS"
304 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
306 echo "start $SINGLEMDS with disabling OI scrub"
307 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
308 error "(7) Fail to start MDS!"
310 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
311 [ "$FLAGS" == "inconsistent" ] ||
312 error "(9) Expect 'inconsistent', but got '$FLAGS'"
314 run_test 1a "Auto trigger initial OI scrub when server mounts"
319 echo "start MDTs without disabling OI scrub"
320 scrub_start_mds 2 "$MOUNT_OPTS_SCRUB"
321 scrub_check_status 3 completed
322 mount_client $MOUNT || error "(4) Fail to start client!"
325 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
330 # OI files to be removed:
332 # idx 2: oi.16.{2,4,8,16,32}
333 # idx 3: oi.16.{3,9,27}
334 for index in 0 2 3; do
336 scrub_remove_ois 1 $index
337 echo "start MDTs with OI scrub disabled"
338 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
339 scrub_check_flags 3 recreated
341 scrub_check_status 5 completed
342 scrub_check_flags 6 ""
345 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
349 scrub_backup_restore 1
350 echo "starting MDTs without disabling OI scrub"
351 scrub_start_mds 2 "$MOUNT_OPTS_SCRUB"
352 scrub_check_status 3 completed
353 mount_client $MOUNT || error "(4) Fail to start client!"
356 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
358 # test_3 is obsolete, it will be covered by test_5.
360 formatall > /dev/null
364 scrub_backup_restore 1
365 echo "starting MDTs with OI scrub disabled"
366 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
367 scrub_check_status 3 init
368 scrub_check_flags 4 inconsistent
370 #run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
374 scrub_backup_restore 1
375 echo "starting MDTs with OI scrub disabled"
376 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
377 scrub_check_flags 4 inconsistent
378 mount_client $MOUNT || error "(5) Fail to start client!"
381 scrub_check_status 7 completed
382 scrub_check_flags 8 ""
384 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
387 formatall > /dev/null
391 scrub_backup_restore 1
392 echo "starting MDTs with OI scrub disabled"
393 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
394 scrub_check_status 3 init
395 scrub_check_flags 4 inconsistent
396 mount_client $MOUNT || error "(5) Fail to start client!"
399 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
400 do_nodes $(comma_list $(mdts_nodes)) \
401 $LCTL set_param fail_val=3 fail_loc=0x190
404 umount_client $MOUNT || error "(7) Fail to stop client!"
405 scrub_check_status 8 scanning
407 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
408 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param fail_loc=0x191
413 do_nodes $(comma_list $(mdts_nodes)) \
414 $LCTL set_param fail_loc=0 fail_val=0
416 echo "starting MDTs with OI scrub disabled"
417 scrub_start_mds 10 "$MOUNT_OPTS_NOSCRUB"
418 scrub_check_status 11 crashed
421 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
422 do_nodes $(comma_list $(mdts_nodes)) \
423 $LCTL set_param fail_val=3 fail_loc=0x190
425 echo "starting MDTs without disabling OI scrub"
426 scrub_start_mds 13 "$MOUNT_OPTS_SCRUB"
427 scrub_check_status 14 scanning
429 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
430 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param fail_loc=0x192
432 scrub_check_status 15 failed
433 mount_client $MOUNT || error "(16) Fail to start client!"
435 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
436 do_nodes $(comma_list $(mdts_nodes)) \
437 $LCTL set_param fail_val=3 fail_loc=0x190
440 for n in $(seq $MDSCOUNT); do
441 stat $DIR/$tdir/mds$n/${tfile}800 ||
442 error "(17) Failed to stat mds$n/${tfile}800"
445 scrub_check_status 18 scanning
447 do_nodes $(comma_list $(mdts_nodes)) \
448 $LCTL set_param fail_loc=0 fail_val=0
450 scrub_check_status 19 completed
451 scrub_check_flags 20 ""
453 run_test 5 "OI scrub state machine"
457 scrub_backup_restore 1
458 echo "starting MDTs with OI scrub disabled"
459 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
460 scrub_check_flags 4 inconsistent
461 mount_client $MOUNT || error "(5) Fail to start client!"
464 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
465 do_nodes $(comma_list $(mdts_nodes)) \
466 $LCTL set_param fail_val=3 fail_loc=0x190
470 # Sleep 5 sec to guarantee at least one object processed by OI scrub
472 # Fail the OI scrub to guarantee there is at least one checkpoint
473 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
474 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param fail_loc=0x192
476 scrub_check_status 7 failed
478 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
479 do_nodes $(comma_list $(mdts_nodes)) \
480 $LCTL set_param fail_val=3 fail_loc=0x190
483 for n in $(seq $MDSCOUNT); do
484 # stat will re-trigger OI scrub
485 stat $DIR/$tdir/mds$n/${tfile}800 ||
486 error "(8) Failed to stat mds$n/${tfile}800"
489 umount_client $MOUNT || error "(9) Fail to stop client!"
490 scrub_check_status 10 scanning
492 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
493 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param fail_loc=0x191
497 for n in $(seq $MDSCOUNT); do
498 position0[$n]=$(scrub_status $n |
499 awk '/^last_checkpoint_position/ {print $2}')
500 position0[$n]=$((${position0[$n]} + 1))
505 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
506 do_nodes $(comma_list $(mdts_nodes)) \
507 $LCTL set_param fail_val=3 fail_loc=0x190
509 echo "starting MDTs without disabling OI scrub"
510 scrub_start_mds 12 "$MOUNT_OPTS_SCRUB"
512 scrub_check_status 13 scanning
515 for n in $(seq $MDSCOUNT); do
516 position1[$n]=$(scrub_status $n |
517 awk '/^latest_start_position/ {print $2}')
518 if [ ${position0[$n]} -ne ${position1[$n]} ]; then
519 error "(14) Expected position ${position0[$n]}, but" \
520 "got ${position1[$n]}"
524 do_nodes $(comma_list $(mdts_nodes)) \
525 $LCTL set_param fail_loc=0 fail_val=0
527 scrub_check_status 15 completed
528 scrub_check_flags 16 ""
530 run_test 6 "OI scrub resumes from last checkpoint"
534 scrub_backup_restore 1
535 echo "starting MDTs with OI scrub disabled"
536 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
537 scrub_check_flags 4 inconsistent
538 mount_client $MOUNT || error "(5) Fail to start client!"
541 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
542 do_nodes $(comma_list $(mdts_nodes)) \
543 $LCTL set_param fail_val=3 fail_loc=0x190
548 for n in $(seq $MDSCOUNT); do
549 stat $DIR/$tdir/mds$n/${tfile}300 ||
550 error "(7) Failed to stat mds$n/${tfile}300!"
553 scrub_check_status 8 scanning
554 scrub_check_flags 9 inconsistent,auto
556 do_nodes $(comma_list $(mdts_nodes)) \
557 $LCTL set_param fail_loc=0 fail_val=0
559 scrub_check_status 10 completed
562 run_test 7 "System is available during OI scrub scanning"
566 scrub_backup_restore 1
567 echo "starting MDTs with OI scrub disabled"
568 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
569 scrub_check_flags 4 inconsistent
571 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
572 do_nodes $(comma_list $(mdts_nodes)) \
573 $LCTL set_param fail_val=1 fail_loc=0x190
576 scrub_check_status 6 scanning
578 scrub_check_status 8 stopped
580 scrub_check_status 10 scanning
582 do_nodes $(comma_list $(mdts_nodes)) \
583 $LCTL set_param fail_loc=0 fail_val=0
585 scrub_check_status 11 completed
586 scrub_check_flags 12 ""
588 run_test 8 "Control OI scrub manually"
591 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
592 skip "Testing on UP system, the speed may be inaccurate."
597 scrub_backup_restore 1
599 echo "starting MDTs with OI scrub disabled"
600 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
601 scrub_check_flags 4 inconsistent
603 local BASE_SPEED1=100
605 # OI scrub should run with full speed under inconsistent case
606 scrub_start 5 -s $BASE_SPEED1
609 scrub_check_status 6 completed
610 scrub_check_flags 7 ""
611 # OI scrub should run with limited speed under non-inconsistent case
612 scrub_start 8 -s $BASE_SPEED1 -r
615 scrub_check_status 9 scanning
617 # Do NOT ignore that there are 1024 pre-fetched items. And there
618 # may be time error, normally it should be less than 2 seconds.
619 # We allow another 20% schedule error.
620 local PRE_FETCHED=1024
622 # MAX_MARGIN = 1.2 = 12 / 10
623 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
624 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
626 for n in $(seq $MDSCOUNT); do
627 local SPEED=$(scrub_status $n | \
628 awk '/^average_speed/ { print $2 }')
629 [ $SPEED -lt $MAX_SPEED ] ||
630 error "(10) Got speed $SPEED, expected less than" \
635 local BASE_SPEED2=300
637 for n in $(seq $MDSCOUNT); do
638 do_facet mds$n $LCTL set_param -n \
639 mdd.$(facet_svc mds$n).lfsck_speed_limit $BASE_SPEED2
643 # MIN_MARGIN = 0.8 = 8 / 10
644 local MIN_SPEED=$(((PRE_FETCHED + \
645 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
646 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
647 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
648 # MAX_MARGIN = 1.2 = 12 / 10
649 MAX_SPEED=$(((PRE_FETCHED + \
650 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
651 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
652 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
653 for n in $(seq $MDSCOUNT); do
654 SPEED=$(scrub_status $n | awk '/^average_speed/ { print $2 }')
655 [ $SPEED -gt $MIN_SPEED ] ||
656 error "(11) Got speed $SPEED, expected more than" \
658 [ $SPEED -lt $MAX_SPEED ] ||
659 error "(12) Got speed $SPEED, expected less than" \
662 do_facet mds$n $LCTL set_param -n \
663 mdd.$(facet_svc mds$n).lfsck_speed_limit 0
666 scrub_check_status 13 completed
668 run_test 9 "OI scrub speed control"
672 scrub_backup_restore 1
673 echo "starting mds$n with OI scrub disabled"
674 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
675 scrub_check_flags 4 inconsistent
676 mount_client $MOUNT || error "(5) Fail to start client!"
679 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
680 do_nodes $(comma_list $(mdts_nodes)) \
681 $LCTL set_param fail_val=1 fail_loc=0x190
684 scrub_check_status 7 scanning
685 umount_client $MOUNT || error "(8) Fail to stop client!"
687 echo "starting MDTs with OI scrub disabled"
688 scrub_start_mds 10 "$MOUNT_OPTS_NOSCRUB"
689 scrub_check_status 11 paused
691 echo "starting MDTs without disabling OI scrub"
692 scrub_start_mds 13 "$MOUNT_OPTS_SCRUB"
693 scrub_check_status 14 scanning
695 do_nodes $(comma_list $(mdts_nodes)) \
696 $LCTL set_param fail_loc=0 fail_val=0
698 scrub_check_status 15 completed
699 scrub_check_flags 16 ""
701 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
703 # test_10b is obsolete, it will be coverded by related sanity-lfsck tests.
706 scrub_backup_restore 1
707 echo "starting MDTs with OI scrub disabled"
708 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
709 scrub_check_flags 4 inconsistent
711 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
712 do_nodes $(comma_list $(mdts_nodes)) \
713 $LCTL set_param fail_val=3 fail_loc=0x190
716 scrub_check_status 6 scanning
718 echo "starting MDTs with OI scrub disabled"
719 scrub_start_mds 8 "$MOUNT_OPTS_NOSCRUB"
720 scrub_check_status 9 paused
722 echo "starting MDTs without disabling OI scrub"
723 scrub_start_mds 11 "$MOUNT_OPTS_SCRUB"
724 scrub_check_status 12 scanning
726 do_nodes $(comma_list $(mdts_nodes)) \
727 $LCTL set_param fail_loc=0 fail_val=0
729 scrub_check_status 13 completed
730 scrub_check_flags 14 ""
732 #run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
740 for n in $(seq $MDSCOUNT); do
741 $LFS mkdir -i $((n - 1)) $DIR/$tdir/mds$n ||
742 error "(1) Fail to mkdir $DIR/$tdir/mds$n"
744 createmany -o $DIR/$tdir/mds$n/f $CREATED ||
745 error "(2) Fail to create under $tdir/mds$n"
748 # reset OI scrub start point by force
750 scrub_check_status 4 completed
755 # OI scrub should skip the new created objects for the first accessing
756 # notice we're creating a new llog for every OST on every startup
757 # new features can make this even less stable, so we only check
758 # that the number of skipped files is less than 2x the number of files
759 local MAXIMUM=$((CREATED * 2))
760 local MINIMUM=$((CREATED + 1)) # files + directory
761 for n in $(seq $MDSCOUNT); do
762 local SKIPPED=$(scrub_status $n | awk '/^noscrub/ { print $2 }')
763 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
764 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects" \
765 "skipped on mds$n, but got $SKIPPED"
767 checked0[$n]=$(scrub_status $n | awk '/^checked/ { print $2 }')
770 # reset OI scrub start point by force
772 scrub_check_status 7 completed
774 # OI scrub should skip the new created object only once
775 for n in $(seq $MDSCOUNT); do
776 SKIPPED=$(scrub_status $n | awk '/^noscrub/ { print $2 }')
777 checked1[$n]=$(scrub_status $n | awk '/^checked/ { print $2 }')
779 [ ${checked0[$n]} -ne ${checked1[$n]} -o $SKIPPED -eq 0 ] ||
780 error "(8) Expect 0 objects skipped on mds$n, but" \
784 run_test 11 "OI scrub skips the new created objects only once"
788 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
790 local count=$(precreated_ost_obj_count 0 0)
792 #define OBD_FAIL_OSD_COMPAT_INVALID_ENTRY 0x195
793 do_facet ost1 $LCTL set_param fail_loc=0x195
794 createmany -o $DIR/$tdir/f $((count + 32))
796 umount_client $MOUNT || error "(1) Fail to stop client!"
798 stop ost1 || error "(2) Fail to stop ost1"
800 #define OBD_FAIL_OST_NODESTROY 0x233
801 do_facet ost1 $LCTL set_param fail_loc=0x233
803 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB ||
804 error "(3) Fail to start ost1"
806 mount_client $MOUNT || error "(4) Fail to start client!"
808 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail"
810 $START_SCRUB_ON_OST -r || error "(6) Fail to start OI scrub on OST!"
812 do_facet ost1 $LCTL set_param fail_loc=0
813 wait_update_facet ost1 "$LCTL get_param -n \
814 osd-ldiskfs.$(facet_svc ost1).oi_scrub |
815 awk '/^status/ { print \\\$2 }'" "completed" 6 ||
816 error "(7) Expected '$expected' on ost1"
818 ls -ail $DIR/$tdir > /dev/null || {
820 error "(8) ls should succeed"
823 run_test 12 "OI scrub can rebuild invalid /O entries"
827 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
829 local count=$(precreated_ost_obj_count 0 0)
831 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
832 do_facet ost1 $LCTL set_param fail_loc=0x196
833 createmany -o $DIR/$tdir/f $((count + 32))
834 do_facet ost1 $LCTL set_param fail_loc=0
836 umount_client $MOUNT || error "(1) Fail to stop client!"
838 stop ost1 || error "(2) Fail to stop ost1"
840 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB ||
841 error "(3) Fail to start ost1"
843 mount_client $MOUNT || error "(4) Fail to start client!"
845 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail"
847 $START_SCRUB_ON_OST -r || error "(6) Fail to start OI scrub on OST!"
849 wait_update_facet ost1 "$LCTL get_param -n \
850 osd-ldiskfs.$(facet_svc ost1).oi_scrub |
851 awk '/^status/ { print \\\$2 }'" "completed" 6 ||
852 error "(7) Expected '$expected' on ost1"
854 ls -ail $DIR/$tdir > /dev/null || error "(8) ls should succeed"
856 run_test 13 "OI scrub can rebuild missed /O entries"
860 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
862 local count=$(precreated_ost_obj_count 0 0)
864 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
865 do_facet ost1 $LCTL set_param fail_loc=0x196
866 createmany -o $DIR/$tdir/f $((count + 32))
867 do_facet ost1 $LCTL set_param fail_loc=0
869 umount_client $MOUNT || error "(1) Fail to stop client!"
871 stop ost1 || error "(2) Fail to stop ost1"
874 run_e2fsck $(facet_host ost1) $(ostdevname 1) "-y" ||
875 error "(3) Fail to run e2fsck error"
877 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
878 error "(4) Fail to start ost1"
880 mount_client $MOUNT || error "(5) Fail to start client!"
882 local LF_REPAIRED=$($SHOW_SCRUB_ON_OST |
883 awk '/^lf_reparied/ { print $2 }')
884 [ $LF_REPAIRED -gt 0 ] ||
885 error "(6) Some entry under /lost+found should be repaired"
887 ls -ail $DIR/$tdir > /dev/null || error "(7) ls should succeed"
889 run_test 14 "OI scrub can repair objects under lost+found"
892 # skip test_15 for LU-4182
893 [ $MDSCOUNT -ge 2 ] && skip "skip now for >= 2 MDTs" && return
894 local server_version=$(lustre_version_code $SINGLEMDS)
896 scrub_backup_restore 1
897 echo "starting MDTs with OI scrub disabled"
898 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
899 scrub_check_status 3 init
900 scrub_check_flags 4 inconsistent
902 # run under dryrun mode
903 if [ $server_version -lt $(version_code 2.5.58) ]; then
904 scrub_start 5 --dryrun on
906 scrub_start 5 --dryrun
908 scrub_check_status 6 completed
909 scrub_check_flags 7 inconsistent
910 scrub_check_params 8 dryrun
911 scrub_check_repaired 9 20
913 # run under dryrun mode again
914 if [ $server_version -lt $(version_code 2.5.58) ]; then
915 scrub_start 10 --dryrun on
917 scrub_start 10 --dryrun
919 scrub_check_status 11 completed
920 scrub_check_flags 12 inconsistent
921 scrub_check_params 13 dryrun
922 scrub_check_repaired 14 20
924 # run under normal mode
926 # Lustre-2.x (x <= 5) used "-n off" to disable dryrun which does not
927 # work under Lustre-2.y (y >= 6), the test script should be fixed as
928 # "-noff" or "--dryrun=off" or nothing by default.
929 if [ $server_version -lt $(version_code 2.5.58) ]; then
930 scrub_start 15 --dryrun off
934 scrub_check_status 16 completed
935 scrub_check_flags 17 ""
936 scrub_check_params 18 ""
937 scrub_check_repaired 19 20
939 # run under normal mode again
940 if [ $server_version -lt $(version_code 2.5.58) ]; then
941 scrub_start 20 --dryrun off
945 scrub_check_status 21 completed
946 scrub_check_flags 22 ""
947 scrub_check_params 23 ""
948 scrub_check_repaired 24 0
950 run_test 15 "Dryrun mode OI scrub"
952 # restore MDS/OST size
953 MDSSIZE=${SAVED_MDSSIZE}
954 OSTSIZE=${SAVED_OSTSIZE}
955 OSTCOUNT=${SAVED_OSTCOUNT}
957 # cleanup the system at last