3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [ $(facet_fstype $SINGLEMDS) != "ldiskfs" ] &&
33 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
35 [ $(facet_fstype ost1) != "ldiskfs" ] &&
36 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
38 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
39 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
42 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
43 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
45 [[ $(lustre_version_code ost1) -lt $(version_code 2.4.50) ]] &&
46 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14"
50 MDT_DEV="${FSNAME}-MDT0000"
51 OST_DEV="${FSNAME}-OST0000"
52 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
58 for n in $(seq $MDSCOUNT); do
59 do_facet mds$n $LCTL lfsck_start -M $(facet_svc mds$n) "$@" ||
60 error "($error_id) Failed to start OI scrub on mds$n"
68 for n in $(seq $MDSCOUNT); do
69 do_facet mds$n $LCTL lfsck_stop -M $(facet_svc mds$n) ||
70 error "($error_id) Failed to stop OI scrub on mds$n"
77 do_facet mds$n $LCTL get_param -n \
78 osd-ldiskfs.$(facet_svc mds$n).oi_scrub
81 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
82 START_SCRUB_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV}"
83 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
84 SHOW_SCRUB="do_facet $SINGLEMDS \
85 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
86 SHOW_SCRUB_ON_OST="do_facet ost1 \
87 $LCTL get_param -n osd-ldiskfs.${OST_DEV}.oi_scrub"
88 MOUNT_OPTS_SCRUB="-o user_xattr"
89 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
101 for n in $(seq $MDSCOUNT); do
102 echo "creating $nfiles files on mds$n"
103 if [ $n -eq 1 ]; then
104 mkdir -p $DIR/$tdir/mds$n ||
105 error "Failed to create directory mds$n"
107 $LFS mkdir -i $((n - 1)) $DIR/$tdir/mds$n ||
108 error "Failed to create remote directory mds$n"
110 cp $LUSTRE/tests/*.sh $DIR/$tdir/mds$n ||
111 error "Failed to copy files to mds$n"
112 if [[ $nfiles -gt 0 ]]; then
113 createmany -o $DIR/$tdir/mds$n/$tfile $nfiles ||
114 error "createmany failed on mds$n"
118 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
119 for n in $(seq $MDSCOUNT); do
121 stop mds$n > /dev/null || error "Fail to stop MDS$n!"
130 for n in $(seq $MDSCOUNT); do
131 start mds$n $(mdsdevname $n) $opts >/dev/null ||
132 error "($error_id) Failed to start mds$n"
140 for n in $(seq $MDSCOUNT); do
141 echo "stopping mds$n"
142 stop mds$n >/dev/null ||
143 error "($error_id) Failed to stop mds$n"
147 scrub_check_status() {
153 for n in $(seq $MDSCOUNT); do
154 actual=$(do_facet mds$n $LCTL get_param -n \
155 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
156 awk '/^status/ { print $2 }')
157 if [ "$actual" != "$expected" ]; then
158 error "($error_id) Expected '$expected' on mds$n, but" \
164 scrub_check_flags() {
170 for n in $(seq $MDSCOUNT); do
171 actual=$(do_facet mds$n $LCTL get_param -n \
172 osd-ldiskfs.$(facet_svc mds$n).oi_scrub |
173 awk '/^flags/ { print $2 }')
174 if [ "$actual" != "$expected" ]; then
175 error "($error_id) Expected '$expected' on mds$n, but" \
185 for n in $(seq $MDSCOUNT); do
186 diff -q $LUSTRE/tests/test-framework.sh \
187 $DIR/$tdir/mds$n/test-framework.sh ||
188 error "($error_id) File data check failed"
197 for n in $(seq $MDSCOUNT); do
198 mds_remove_ois mds$n $index ||
199 error "($error_id) Failed to remove OI .$index on mds$n"
203 scrub_backup_restore() {
208 for n in $(seq $MDSCOUNT); do
209 mds_backup_restore mds$n $igif ||
210 error "(error_id) Backup/restore on mds$n failed"
214 scrub_enable_auto() {
217 for n in $(seq $MDSCOUNT); do
218 do_facet mds$n $LCTL set_param -n \
219 osd-ldiskfs.$(facet_svc mds$n).auto_scrub 1
225 echo "starting MDTs without disabling OI scrub"
226 scrub_start_mds 1 "$MOUNT_OPTS_SCRUB"
227 scrub_check_status 2 init
228 scrub_check_flags 3 ""
229 mount_client $MOUNT || error "(4) Fail to start client!"
232 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
236 echo "start $SINGLEMDS without disabling OI scrub"
237 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
238 error "(1) Fail to start MDS!"
240 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
241 [ "$STATUS" == "init" ] ||
242 error "(2) Expect 'init', but got '$STATUS'"
244 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
245 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
247 mount_client $MOUNT || error "(4) Fail to start client!"
249 #define OBD_FAIL_OSD_FID_MAPPING 0x193
250 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
251 # update .lustre OI mapping
253 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
255 umount_client $MOUNT || error "(5) Fail to stop client!"
257 echo "stop $SINGLEMDS"
258 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
260 echo "start $SINGLEMDS with disabling OI scrub"
261 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
262 error "(7) Fail to start MDS!"
264 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
265 [ "$STATUS" == "init" ] ||
266 error "(8) Expect 'init', but got '$STATUS'"
268 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
269 [ "$FLAGS" == "inconsistent" ] ||
270 error "(9) Expect 'inconsistent', but got '$FLAGS'"
272 run_test 1a "Auto trigger initial OI scrub when server mounts"
277 echo "start MDTs without disabling OI scrub"
278 scrub_start_mds 2 "$MOUNT_OPTS_SCRUB"
280 scrub_check_status 3 completed
281 mount_client $MOUNT || error "(4) Fail to start client!"
284 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
289 # OI files to be removed:
292 # idx 2: oi.16.{2,4,8,16,32}
293 # idx 3: oi.16.{3,9,27}
294 # idx 5: oi.16.{5,25}
295 # idx 7: oi.16.{7,49}
296 for index in 0 1 2 3 5 7; do
298 scrub_remove_ois 1 $index
300 echo "start MDTs with OI scrub disabled"
301 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
302 scrub_check_flags 3 recreated
305 scrub_check_status 5 completed
306 scrub_check_flags 6 ""
309 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
313 scrub_backup_restore 1
314 echo "starting MDTs without disabling OI scrub"
315 scrub_start_mds 2 "$MOUNT_OPTS_SCRUB"
317 scrub_check_status 3 completed
318 mount_client $MOUNT || error "(4) Fail to start client!"
321 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
325 scrub_backup_restore 1
326 echo "starting MDTs with OI scrub disabled"
327 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
329 scrub_check_status 3 init
330 scrub_check_flags 4 inconsistent
334 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
338 scrub_backup_restore 1
339 echo "starting MDTs with OI scrub disabled"
340 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
341 scrub_check_status 3 init
342 scrub_check_flags 4 inconsistent
343 mount_client $MOUNT || error "(5) Fail to start client!"
347 scrub_check_status 7 completed
349 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
353 scrub_backup_restore 1
354 echo "starting MDTs with OI scrub disabled"
355 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
356 scrub_check_status 3 init
357 scrub_check_flags 4 inconsistent
358 mount_client $MOUNT || error "(5) Fail to start client!"
362 for n in $(seq $MDSCOUNT); do
363 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
364 do_facet mds$n $LCTL set_param fail_val=3
365 do_facet mds$n $LCTL set_param fail_loc=0x190
369 umount_client $MOUNT || error "(7) Fail to stop client!"
371 scrub_check_status 8 scanning
373 for n in $(seq $MDSCOUNT); do
374 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
375 do_facet mds$n $LCTL set_param fail_loc=0x191
380 for n in $(seq $MDSCOUNT); do
381 do_facet mds$n $LCTL set_param fail_loc=0
382 do_facet mds$n $LCTL set_param fail_val=0
385 echo "starting MDTs with OI scrub disabled"
386 scrub_start_mds 10 "$MOUNT_OPTS_NOSCRUB"
388 scrub_check_status 11 crashed
392 for n in $(seq $MDSCOUNT); do
393 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
394 do_facet mds$n $LCTL set_param fail_val=3
395 do_facet mds$n $LCTL set_param fail_loc=0x190
397 echo "starting MDTs without disabling OI scrub"
398 scrub_start_mds 13 "$MOUNT_OPTS_SCRUB"
400 scrub_check_status 14 scanning
402 for n in $(seq $MDSCOUNT); do
403 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
404 do_facet mds$n $LCTL set_param fail_loc=0x192
407 scrub_check_status 15 failed
409 mount_client $MOUNT || error "(16) Fail to start client!"
411 for n in $(seq $MDSCOUNT); do
412 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
413 do_facet mds$n $LCTL set_param fail_val=3
414 do_facet mds$n $LCTL set_param fail_loc=0x190
415 stat $DIR/$tdir/mds$n/${tfile}1000 ||
416 error "(17) Failed to stat mds$n/${tfile}1000"
419 scrub_check_status 18 scanning
421 for n in $(seq $MDSCOUNT); do
422 do_facet mds$n $LCTL set_param fail_loc=0
423 do_facet mds$n $LCTL set_param fail_val=0
426 scrub_check_status 19 completed
428 scrub_check_flags 20 ""
430 run_test 5 "OI scrub state machine"
434 scrub_backup_restore 1
435 echo "starting MDTs with OI scrub disabled"
436 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
437 scrub_check_status 3 init
438 scrub_check_flags 4 inconsistent
439 mount_client $MOUNT || error "(5) Fail to start client!"
442 for n in $(seq $MDSCOUNT); do
443 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
444 do_facet mds$n $LCTL set_param fail_val=3
445 do_facet mds$n $LCTL set_param fail_loc=0x190
449 # Sleep 5 sec to guarantee at least one object processed by OI scrub
451 # Fail the OI scrub to guarantee there is at least one checkpoint
452 for n in $(seq $MDSCOUNT); do
453 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
454 do_facet mds$n $LCTL set_param fail_loc=0x192
457 scrub_check_status 7 failed
459 for n in $(seq $MDSCOUNT); do
460 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
461 do_facet mds$n $LCTL set_param fail_val=3
462 do_facet mds$n $LCTL set_param fail_loc=0x190
463 # stat will re-trigger OI scrub
464 stat $DIR/$tdir/mds$n/${tfile}800 ||
465 error "(8) Failed to stat mds$n/${tfile}800"
468 umount_client $MOUNT || error "(9) Fail to stop client!"
470 scrub_check_status 10 scanning
472 for n in $(seq $MDSCOUNT); do
473 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
474 do_facet mds$n $LCTL set_param fail_loc=0x191
478 for n in $(seq $MDSCOUNT); do
479 position0[$n]=$(scrub_status $n |
480 awk '/^last_checkpoint_position/ {print $2}')
481 position0[$n]=$((${position0[$n]} + 1))
486 for n in $(seq $MDSCOUNT); do
487 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
488 do_facet mds$n $LCTL set_param fail_val=3
489 do_facet mds$n $LCTL set_param fail_loc=0x190
491 echo "starting MDTs without disabling OI scrub"
492 scrub_start_mds 12 "$MOUNT_OPTS_SCRUB"
494 scrub_check_status 13 scanning
497 for n in $(seq $MDSCOUNT); do
498 positions1[$n]=$(scrub_status $n |
499 awk '/^latest_start_position/ {print $2}')
500 if [ ${position0[$n]} -ne ${position1[$n]} ]; then
501 error "(14) Expected position ${position0[$n]}, but" \
502 "got ${position1[$n]}"
506 for n in $(seq $MDSCOUNT); do
507 do_facet mds$n $LCTL set_param fail_loc=0
508 do_facet mds$n $LCTL set_param fail_val=0
511 scrub_check_status 15 completed
513 scrub_check_flags 16 ""
515 run_test 6 "OI scrub resumes from last checkpoint"
519 scrub_backup_restore 1
521 echo "starting MDTs with OI scrub disabled"
522 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
523 scrub_check_status 3 init
524 scrub_check_flags 4 inconsistent
526 mount_client $MOUNT || error "(5) Fail to start client!"
530 for n in $(seq $MDSCOUNT); do
531 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
532 do_facet mds$n $LCTL set_param fail_val=3
533 do_facet mds$n $LCTL set_param fail_loc=0x190
537 for n in $(seq $MDSCOUNT); do
538 stat $DIR/$tdir/mds$n/${tfile}300 ||
539 error "(7) Failed to stat mds$n/${tfile}300!"
542 scrub_check_status 8 scanning
544 scrub_check_flags 9 inconsistent,auto
546 for n in $(seq $MDSCOUNT); do
547 do_facet mds$n $LCTL set_param fail_loc=0
548 do_facet mds$n $LCTL set_param fail_val=0
551 scrub_check_status 10 completed
555 run_test 7 "System is available during OI scrub scanning"
559 scrub_backup_restore 1
561 echo "starting MDTs with OI scrub disabled"
562 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
564 scrub_check_status 3 init
566 scrub_check_flags 4 inconsistent
569 for n in $(seq $MDSCOUNT); do
570 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
571 do_facet mds$n $LCTL set_param fail_val=1
572 do_facet mds$n $LCTL set_param fail_loc=0x190
576 scrub_check_status 6 scanning
580 scrub_check_status 8 stopped
584 scrub_check_status 10 scanning
586 for n in $(seq $MDSCOUNT); do
587 do_facet mds$n $LCTL set_param fail_loc=0
588 do_facet mds$n $LCTL set_param fail_val=0
591 scrub_check_status 11 completed
593 scrub_check_flags 12 ""
595 run_test 8 "Control OI scrub manually"
598 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
599 skip "Testing on UP system, the speed may be inaccurate."
604 scrub_backup_restore 1
606 echo "starting MDTs with OI scrub disabled"
607 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
609 scrub_check_status 3 init
611 scrub_check_flags 4 inconsistent
613 local BASE_SPEED1=100
615 # OI scrub should run with full speed under inconsistent case
616 scrub_start 5 -s $BASE_SPEED1
619 scrub_check_status 6 completed
621 scrub_check_flags 7 ""
623 # OI scrub should run with limited speed under non-inconsistent case
624 scrub_start 8 -s $BASE_SPEED1 -r
627 scrub_check_status 9 scanning
629 # Do NOT ignore that there are 1024 pre-fetched items. And there
630 # may be time error, normally it should be less than 2 seconds.
631 # We allow another 20% schedule error.
632 local PRE_FETCHED=1024
634 # MAX_MARGIN = 1.2 = 12 / 10
635 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
636 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
638 for n in $(seq $MDSCOUNT); do
639 local SPEED=$(scrub_status $n | \
640 awk '/^average_speed/ { print $2 }')
641 [ $SPEED -lt $MAX_SPEED ] ||
642 error "(10) Got speed $SPEED, expected less than" \
647 local BASE_SPEED2=300
649 for n in $(seq $MDSCOUNT); do
650 do_facet mds$n $LCTL set_param -n \
651 mdd.$(facet_svc mds$n).lfsck_speed_limit $BASE_SPEED2
655 # MIN_MARGIN = 0.8 = 8 / 10
656 local MIN_SPEED=$(((PRE_FETCHED + \
657 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
658 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
659 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
660 # MAX_MARGIN = 1.2 = 12 / 10
661 MAX_SPEED=$(((PRE_FETCHED + \
662 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
663 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
664 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
665 for n in $(seq $MDSCOUNT); do
666 SPEED=$(scrub_status $n | awk '/^average_speed/ { print $2 }')
667 [ $SPEED -gt $MIN_SPEED ] ||
668 error "(11) Got speed $SPEED, expected more than" \
670 [ $SPEED -lt $MAX_SPEED ] ||
671 error "(12) Got speed $SPEED, expected less than" \
674 do_facet mds$n $LCTL set_param -n \
675 mdd.$(facet_svc mds$n).lfsck_speed_limit 0
678 scrub_check_status 13 completed
680 run_test 9 "OI scrub speed control"
684 scrub_backup_restore 1
686 echo "starting mds$n with OI scrub disabled"
687 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
689 scrub_check_status 3 init
691 scrub_check_flags 4 inconsistent
693 mount_client $MOUNT || error "(5) Fail to start client!"
697 for n in $(seq $MDSCOUNT); do
698 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
699 do_facet mds$n $LCTL set_param fail_val=1
700 do_facet mds$n $LCTL set_param fail_loc=0x190
704 scrub_check_status 7 scanning
706 umount_client $MOUNT || error "(8) Fail to stop client!"
710 echo "starting MDTs with OI scrub disabled"
711 scrub_start_mds 10 "$MOUNT_OPTS_NOSCRUB"
713 scrub_check_status 11 paused
717 echo "starting MDTs without disabling OI scrub"
718 scrub_start_mds 13 "$MOUNT_OPTS_SCRUB"
720 scrub_check_status 14 scanning
722 for n in $(seq $MDSCOUNT); do
723 do_facet mds$n $LCTL set_param fail_loc=0
724 do_facet mds$n $LCTL set_param fail_val=0
727 scrub_check_status 15 completed
729 scrub_check_flags 16 ""
731 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
733 # test_10b is obsolete, it will be coverded by related sanity-lfsck tests.
736 scrub_backup_restore 1
738 echo "starting MDTs with OI scrub disabled"
739 scrub_start_mds 2 "$MOUNT_OPTS_NOSCRUB"
741 scrub_check_status 3 init
743 scrub_check_flags 4 inconsistent
746 for n in $(seq $MDSCOUNT); do
747 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
748 do_facet mds$n $LCTL set_param fail_val=3
749 do_facet mds$n $LCTL set_param fail_loc=0x190
754 scrub_check_status 6 scanning
758 echo "starting MDTs with OI scrub disabled"
759 scrub_start_mds 8 "$MOUNT_OPTS_NOSCRUB"
761 scrub_check_status 9 paused
765 echo "starting MDTs without disabling OI scrub"
766 scrub_start_mds 11 "$MOUNT_OPTS_SCRUB"
768 scrub_check_status 12 scanning
770 for n in $(seq $MDSCOUNT); do
771 do_facet mds$n $LCTL set_param fail_loc=0
772 do_facet mds$n $LCTL set_param fail_val=0
775 scrub_check_status 13 completed
777 scrub_check_flags 14 ""
779 #run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
785 formatall > /dev/null
790 local tname=`date +%s`
791 rm -rf $MOUNT/$tname > /dev/null
792 mkdir -p $MOUNT/$tname || error "(0) Failed to create $MOUNT/$tname"
794 for n in $(seq $MDSCOUNT); do
795 $LFS mkdir -i $((n - 1)) $MOUNT/$tname/mds$n ||
796 error "(1) Fail to mkdir $MOUNT/$tname/mds$n"
798 createmany -o $MOUNT/$tname/mds$n/f $CREATED ||
799 error "(2) Fail to create in $tname/mds$n"
803 do_facet $SINGLEMDS $LCTL clear
804 start_full_debug_logging
805 # reset OI scrub start point by force
808 scrub_check_status 4 completed
810 # OI scrub should skip the new created objects for the first accessing
811 # notice we're creating a new llog for every OST on every startup
812 # new features can make this even less stable, so we only check
813 # that the number of skipped files is less than 2x the number of files
814 local MAXIMUM=$((CREATED * 2))
815 local MINIMUM=$((CREATED + 1)) # files + directory
816 for n in $(seq $MDSCOUNT); do
817 local SKIPPED=$(scrub_status $n | awk '/^noscrub/ { print $2 }')
818 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
819 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects" \
820 "skipped on mds$n, but got $SKIPPED"
823 # reset OI scrub start point by force
826 scrub_check_status 7 completed
828 # OI scrub should skip the new created object only once
829 for n in $(seq $MDSCOUNT); do
830 SKIPPED=$(scrub_status $n | awk '/^noscrub/ { print $2 }')
831 [ $SKIPPED -eq 0 ] ||
832 error "(8) Expect 0 objects skipped on mds$n, but" \
836 stop_full_debug_logging
837 restore_mount $MOUNT || error "(9) Fail to start client!"
838 rm -rf $MOUNT/$tname > /dev/null
840 run_test 11 "OI scrub skips the new created objects only once"
846 formatall > /dev/null
851 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
853 #define OBD_FAIL_OSD_COMPAT_INVALID_ENTRY 0x195
854 do_facet ost1 $LCTL set_param fail_loc=0x195
855 createmany -o $DIR/$tdir/f 1000
862 do_facet ost1 $LCTL set_param fail_loc=0
863 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
864 [ "$STATUS" == "init" ] ||
865 error "(1) Expect 'init', but got '$STATUS'"
867 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
870 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
871 [ "$STATUS" == "completed" ] ||
872 error "(3) Expect 'completed', but got '$STATUS'"
874 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(4) ls should succeed"
876 run_test 12 "OI scrub can rebuild invalid /O entries"
882 formatall > /dev/null
887 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
889 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
890 do_facet ost1 $LCTL set_param fail_loc=0x196
891 createmany -o $DIR/$tdir/f 1000
892 do_facet ost1 $LCTL set_param fail_loc=0
899 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
900 [ "$STATUS" == "init" ] ||
901 error "(1) Expect 'init', but got '$STATUS'"
903 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
905 $START_SCRUB_ON_OST || error "(3) Fail to start OI scrub on OST!"
907 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
908 [ "$STATUS" == "completed" ] ||
909 error "(4) Expect 'completed', but got '$STATUS'"
911 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(5) ls should succeed"
913 run_test 13 "OI scrub can rebuild missed /O entries"
919 formatall > /dev/null
924 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
926 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
927 do_facet ost1 $LCTL set_param fail_loc=0x196
928 createmany -o $DIR/$tdir/f 64
929 do_facet ost1 $LCTL set_param fail_loc=0
936 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
937 [ "$STATUS" == "init" ] ||
938 error "(1) Expect 'init', but got '$STATUS'"
940 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
946 run_e2fsck $(facet_host ost1) $(ostdevname 1) "-y" ||
947 error "(3) Fail to run e2fsck error"
952 local LF_REPAIRED=$($SHOW_SCRUB_ON_OST |
953 awk '/^lf_reparied/ { print $2 }')
954 [ $LF_REPAIRED -gt 0 ] ||
955 error "(4) Some entry under /lost+found should be repaired"
957 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(5) ls should succeed"
959 run_test 14 "OI scrub can repair objects under lost+found"
961 # restore MDS/OST size
962 MDSSIZE=${SAVED_MDSSIZE}
963 OSTSIZE=${SAVED_OSTSIZE}
965 # cleanup the system at last