3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
29 check_and_setup_lustre
31 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
32 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
34 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
35 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
40 MDT_DEV="${FSNAME}-MDT0000"
41 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
42 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
43 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
44 SHOW_SCRUB="do_facet $SINGLEMDS \
45 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
46 MOUNT_OPTS_SCRUB="-o user_xattr"
47 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
57 echo "preparing... ${nfiles} files will be created."
59 cp $LUSTRE/tests/*.sh $DIR/$tdir/
60 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
61 error "createmany failed"; }
64 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
65 echo "stop $SINGLEMDS"
66 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
71 echo "start $SINGLEMDS without disabling OI scrub"
72 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
73 error "(1) Fail to start MDS!"
75 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
76 [ "$STATUS" == "init" ] ||
77 error "(2) Expect 'init', but got '$STATUS'"
79 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
80 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
82 mount_client $MOUNT || error "(4) Fail to start client!"
84 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
85 error "(5) File diff failed unexpected!"
87 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
91 echo "start $SINGLEMDS without disabling OI scrub"
92 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
93 error "(1) Fail to start MDS!"
95 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
96 [ "$STATUS" == "init" ] ||
97 error "(2) Expect 'init', but got '$STATUS'"
99 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
100 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
102 mount_client $MOUNT || error "(4) Fail to start client!"
104 #define OBD_FAIL_OSD_FID_MAPPING 0x193
105 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
106 # update .lustre OI mapping
108 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
110 umount_client $MOUNT || error "(5) Fail to stop client!"
112 echo "stop $SINGLEMDS"
113 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
115 echo "start $SINGLEMDS with disabling OI scrub"
116 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
117 error "(7) Fail to start MDS!"
119 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
120 [ "$STATUS" == "init" ] ||
121 error "(8) Expect 'init', but got '$STATUS'"
123 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
124 [ "$FLAGS" == "inconsistent" ] ||
125 error "(9) Expect 'inconsistent', but got '$FLAGS'"
127 run_test 1a "Auto trigger initial OI scrub when server mounts"
131 mds_remove_ois || error "(1) Fail to remove/recreate!"
133 echo "start $SINGLEMDS without disabling OI scrub"
134 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
135 error "(2) Fail to start MDS!"
138 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
139 [ "$STATUS" == "completed" ] ||
140 error "(3) Expect 'completed', but got '$STATUS'"
142 mount_client $MOUNT || error "(4) Fail to start client!"
144 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
145 error "(5) File diff failed unexpected!"
147 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
152 # OI files to be removed:
155 # idx 2: oi.16.{2,4,8,16,32}
156 # idx 3: oi.16.{3,9,27}
157 # idx 5: oi.16.{5,25}
158 # idx 7: oi.16.{7,49}
159 for index in 0 1 2 3 5 7; do
161 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
163 echo "start $SINGLEMDS with disabling OI scrub"
164 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
165 /dev/null || error "(2) Fail to start MDS!"
167 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
168 [ "$FLAGS" == "recreated" ] ||
169 error "(3) Expect 'recreated', but got '$FLAGS'"
171 $START_SCRUB || error "(4) Fail to start OI scrub!"
173 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
174 [ "$STATUS" == "completed" ] ||
175 error "(5) Expect 'completed', but got '$STATUS'"
177 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
179 error "(6) Expect empty flags, but got '$FLAGS'"
182 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
186 mds_backup_restore || error "(1) Fail to backup/restore!"
188 echo "start $SINGLEMDS without disabling OI scrub"
189 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
190 error "(2) Fail to start MDS!"
193 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
194 [ "$STATUS" == "completed" ] ||
195 error "(3) Expect 'completed', but got '$STATUS'"
197 mount_client $MOUNT || error "(4) Fail to start client!"
199 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
200 error "(5) File diff failed unexpected!"
202 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
206 mds_backup_restore || error "(1) Fail to backup/restore!"
208 echo "start $SINGLEMDS with disabling OI scrub"
209 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
210 error "(2) Fail to start MDS!"
212 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
213 [ "$STATUS" == "init" ] ||
214 error "(3) Expect 'init', but got '$STATUS'"
216 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
217 [ "$FLAGS" == "inconsistent" ] ||
218 error "(4) Expect 'inconsistent', but got '$FLAGS'"
222 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
226 mds_backup_restore || error "(1) Fail to backup/restore!"
228 echo "start $SINGLEMDS with disabling OI scrub"
229 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
230 error "(2) Fail to start MDS!"
232 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
233 [ "$STATUS" == "init" ] ||
234 error "(3) Expect 'init', but got '$STATUS'"
236 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
237 [ "$FLAGS" == "inconsistent" ] ||
238 error "(4) Expect 'inconsistent', but got '$FLAGS'"
240 mount_client $MOUNT || error "(5) Fail to start client!"
242 do_facet $SINGLEMDS \
243 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
244 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
245 error "(6) File diff failed unexpected!"
248 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
249 [ "$STATUS" == "completed" ] ||
250 error "(7) Expect 'completed', but got '$STATUS'"
252 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
256 mds_backup_restore || error "(1) Fail to backup/restore!"
258 echo "start $SINGLEMDS with disabling OI scrub"
259 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
260 error "(2) Fail to start MDS!"
262 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
263 [ "$STATUS" == "init" ] ||
264 error "(3) Expect 'init', but got '$STATUS'"
266 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
267 [ "$FLAGS" == "inconsistent" ] ||
268 error "(4) Expect 'inconsistent', but got '$FLAGS'"
270 mount_client $MOUNT || error "(5) Fail to start client!"
272 do_facet $SINGLEMDS \
273 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
274 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
275 do_facet $SINGLEMDS $LCTL set_param fail_val=3
276 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
277 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
278 error "(6) File diff failed unexpected!"
280 umount_client $MOUNT || error "(7) Fail to stop client!"
282 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
283 [ "$STATUS" == "scanning" ] ||
284 error "(8) Expect 'scanning', but got '$STATUS'"
286 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
287 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
289 echo "stop $SINGLEMDS"
290 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
292 echo "start $SINGLEMDS with disabling OI scrub"
293 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
294 error "(10) Fail to start MDS!"
296 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
297 [ "$STATUS" == "crashed" ] ||
298 error "(11) Expect 'crashed', but got '$STATUS'"
300 echo "stop $SINGLEMDS"
301 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
303 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
304 do_facet $SINGLEMDS $LCTL set_param fail_val=3
305 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
306 echo "start $SINGLEMDS without disabling OI scrub"
307 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
308 error "(13) Fail to start MDS!"
310 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
311 [ "$STATUS" == "scanning" ] ||
312 error "(14) Expect 'scanning', but got '$STATUS'"
314 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
315 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
317 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
318 [ "$STATUS" == "failed" ] ||
319 error "(15) Expect 'failed', but got '$STATUS'"
321 mount_client $MOUNT || error "(16) Fail to start client!"
323 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
324 do_facet $SINGLEMDS $LCTL set_param fail_val=3
325 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
326 stat $DIR/$tdir/${tfile}1000 ||
327 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
329 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
330 [ "$STATUS" == "scanning" ] ||
331 error "(18) Expect 'scanning', but got '$STATUS'"
333 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
334 do_facet $SINGLEMDS $LCTL set_param fail_val=0
336 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
337 [ "$STATUS" == "completed" ] ||
338 error "(19) Expect 'completed', but got '$STATUS'"
340 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
341 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
343 run_test 5 "OI scrub state machine"
347 mds_backup_restore || error "(1) Fail to backup/restore!"
349 echo "start $SINGLEMDS with disabling OI scrub"
350 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
351 error "(2) Fail to start MDS!"
353 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
354 [ "$STATUS" == "init" ] ||
355 error "(3) Expect 'init', but got '$STATUS'"
357 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
358 [ "$FLAGS" == "inconsistent" ] ||
359 error "(4) Expect 'inconsistent', but got '$FLAGS'"
361 mount_client $MOUNT || error "(5) Fail to start client!"
363 do_facet $SINGLEMDS \
364 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
365 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
366 do_facet $SINGLEMDS $LCTL set_param fail_val=3
367 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
368 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
369 error "(6) File diff failed unexpected!"
371 # Sleep 5 sec to guarantee at least one object processed by OI scrub
373 # Fail the OI scrub to guarantee there is at least one checkpoint
374 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
375 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
377 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
378 [ "$STATUS" == "failed" ] ||
379 error "(7) Expect 'failed', but got '$STATUS'"
381 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
382 do_facet $SINGLEMDS $LCTL set_param fail_val=3
383 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
384 # stat will re-trigger OI scrub
385 stat $DIR/$tdir/${tfile}800 ||
386 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
388 umount_client $MOUNT || error "(9) Fail to stop client!"
390 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
391 [ "$STATUS" == "scanning" ] ||
392 error "(10) Expect 'scanning', but got '$STATUS'"
394 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
395 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
397 local POSITION0=$($SHOW_SCRUB |
398 awk '/^last_checkpoint_position/ {print $2}')
399 POSITION0=$((POSITION0 + 1))
401 echo "stop $SINGLEMDS"
402 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
404 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
405 do_facet $SINGLEMDS $LCTL set_param fail_val=3
406 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
407 echo "start $SINGLEMDS without disabling OI scrub"
408 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
409 error "(12) Fail to start MDS!"
411 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
412 [ "$STATUS" == "scanning" ] ||
413 error "(13) Expect 'scanning', but got '$STATUS'"
415 local POSITION1=$($SHOW_SCRUB |
416 awk '/^latest_start_position/ {print $2}')
417 [ $POSITION0 -eq $POSITION1 ] ||
418 error "(14) Expect position: $POSITION0, but got $POSITION1"
420 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
421 do_facet $SINGLEMDS $LCTL set_param fail_val=0
423 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
424 [ "$STATUS" == "completed" ] ||
425 error "(15) Expect 'completed', but got '$STATUS'"
427 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
428 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
430 run_test 6 "OI scrub resumes from last checkpoint"
434 mds_backup_restore || error "(1) Fail to backup/restore!"
436 echo "start $SINGLEMDS with disabling OI scrub"
437 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
438 error "(2) Fail to start MDS!"
440 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
441 [ "$STATUS" == "init" ] ||
442 error "(3) Expect 'init', but got '$STATUS'"
444 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
445 [ "$FLAGS" == "inconsistent" ] ||
446 error "(4) Expect 'inconsistent', but got '$FLAGS'"
448 mount_client $MOUNT || error "(5) Fail to start client!"
450 do_facet $SINGLEMDS \
451 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
452 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
453 do_facet $SINGLEMDS $LCTL set_param fail_val=3
454 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
455 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
456 error "(6) File diff failed unexpected!"
458 stat $DIR/$tdir/${tfile}300 ||
459 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
461 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
462 [ "$STATUS" == "scanning" ] ||
463 error "(8) Expect 'scanning', but got '$STATUS'"
465 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
466 [ "$FLAGS" == "inconsistent,auto" ] ||
467 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
469 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
470 do_facet $SINGLEMDS $LCTL set_param fail_val=0
472 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
473 [ "$STATUS" == "completed" ] ||
474 error "(10) Expect 'completed', but got '$STATUS'"
476 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
477 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
479 run_test 7 "System is available during OI scrub scanning"
483 mds_backup_restore || error "(1) Fail to backup/restore!"
485 echo "start $SINGLEMDS with disabling OI scrub"
486 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
487 error "(2) Fail to start MDS!"
489 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
490 [ "$STATUS" == "init" ] ||
491 error "(3) Expect 'init', but got '$STATUS'"
493 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
494 [ "$FLAGS" == "inconsistent" ] ||
495 error "(4) Expect 'inconsistent', but got '$FLAGS'"
497 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
498 do_facet $SINGLEMDS $LCTL set_param fail_val=3
499 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
500 $START_SCRUB || error "(5) Fail to start OI scrub!"
502 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
503 [ "$STATUS" == "scanning" ] ||
504 error "(6) Expect 'scanning', but got '$STATUS'"
506 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
508 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
509 [ "$STATUS" == "stopped" ] ||
510 error "(8) Expect 'stopped', but got '$STATUS'"
512 $START_SCRUB || error "(9) Fail to start OI scrub!"
514 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
515 [ "$STATUS" == "scanning" ] ||
516 error "(10) Expect 'scanning', but got '$STATUS'"
518 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
519 do_facet $SINGLEMDS $LCTL set_param fail_val=0
521 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
522 [ "$STATUS" == "completed" ] ||
523 error "(11) Expect 'completed', but got '$STATUS'"
525 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
526 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
528 run_test 8 "Control OI scrub manually"
531 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
532 skip "Testing on UP system, the speed may be inaccurate."
537 mds_backup_restore || error "(1) Fail to backup/restore!"
539 echo "start $SINGLEMDS with disabling OI scrub"
540 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
541 error "(2) Fail to start MDS!"
543 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
544 [ "$STATUS" == "init" ] ||
545 error "(3) Expect 'init', but got '$STATUS'"
547 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
548 [ "$FLAGS" == "inconsistent" ] ||
549 error "(4) Expect 'inconsistent', but got '$FLAGS'"
551 # OI scrub should run with full speed under inconsistent case
552 $START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
555 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
556 [ "$STATUS" == "completed" ] ||
557 error "(6) Expect 'completed', but got '$STATUS'"
559 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
560 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
562 # OI scrub should run with limited speed under non-inconsistent case
563 $START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
566 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
567 [ "$STATUS" == "scanning" ] ||
568 error "(9) Expect 'scanning', but got '$STATUS'"
570 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
571 # Do NOT ignore that there are 1024 pre-fetched items.
572 # And there may be time error, normally it should be less than 2.
573 # We allow another 10% schedule error.
575 # SPEED1 = (pre-fetched + 100 * (time - 2)) / time * 1.1
576 local SPEED1=$(((1024 + 100 * (10 + 2)) / 10 * 11 / 10))
577 [ $SPEED -lt $SPEED1 ] ||
578 error "(10) Got speed $SPEED, expected less than $SPEED1"
581 do_facet $SINGLEMDS \
582 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
585 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
586 # SPEED1 = (pre-fetched + 100 * (time1 - 2) + 300 * (time2 - 2)) / \
587 # (time1 + time2) * 0.9
588 SPEED1=$(((1024 + 100 * (10 - 2) + 300 * (10 - 2)) / \
590 [ $SPEED -gt $SPEED1 ] ||
591 error "(11) Got speed $SPEED, expected more than $SPEED1"
593 # SPEED1 = (pre-fetched + 100 * (time1 + 2) + 300 * (time2 + 2)) / \
594 # (time1 + time2) * 1.1
595 SPEED1=$(((1024 + 100 * (10 + 2) + 300 * (10 + 2)) / \
596 (10 + 10) * 11 / 10))
597 [ $SPEED -lt $SPEED1 ] ||
598 error "(12) Got speed $SPEED, expected less than $SPEED1"
600 do_facet $SINGLEMDS \
601 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
603 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
604 [ "$STATUS" == "completed" ] ||
605 error "(13) Expect 'completed', but got '$STATUS'"
607 run_test 9 "OI scrub speed control"
611 mds_backup_restore || error "(1) Fail to backup/restore!"
613 echo "start $SINGLEMDS with disabling OI scrub"
614 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
615 error "(2) Fail to start MDS!"
617 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
618 [ "$STATUS" == "init" ] ||
619 error "(3) Expect 'init', but got '$STATUS'"
621 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
622 [ "$FLAGS" == "inconsistent" ] ||
623 error "(4) Expect 'inconsistent', but got '$FLAGS'"
625 mount_client $MOUNT || error "(5) Fail to start client!"
627 do_facet $SINGLEMDS \
628 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
629 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
630 do_facet $SINGLEMDS $LCTL set_param fail_val=3
631 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
632 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
633 error "(6) File diff failed unexpected!"
635 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
636 [ "$STATUS" == "scanning" ] ||
637 error "(7) Expect 'scanning', but got '$STATUS'"
639 umount_client $MOUNT || error "(8) Fail to stop client!"
641 echo "stop $SINGLEMDS"
642 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
644 echo "start $SINGLEMDS with disabling OI scrub"
645 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
646 error "(10) Fail to start MDS!"
648 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "paused" ] ||
650 error "(11) Expect 'paused', but got '$STATUS'"
652 echo "stop $SINGLEMDS"
653 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
655 echo "start $SINGLEMDS without disabling OI scrub"
656 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
657 error "(13) Fail to start MDS!"
659 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
660 [ "$STATUS" == "scanning" ] ||
661 error "(14) Expect 'scanning', but got '$STATUS'"
663 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
664 do_facet $SINGLEMDS $LCTL set_param fail_val=0
666 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
667 [ "$STATUS" == "completed" ] ||
668 error "(15) Expect 'completed', but got '$STATUS'"
670 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
671 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
673 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
677 mds_backup_restore || error "(1) Fail to backup/restore!"
679 echo "start $SINGLEMDS with disabling OI scrub"
680 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
681 error "(2) Fail to start MDS!"
683 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
684 [ "$STATUS" == "init" ] ||
685 error "(3) Expect 'init', but got '$STATUS'"
687 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
688 [ "$FLAGS" == "inconsistent" ] ||
689 error "(4) Expect 'inconsistent', but got '$FLAGS'"
691 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
692 do_facet $SINGLEMDS $LCTL set_param fail_val=3
693 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
695 $START_SCRUB || error "(5) Fail to start OI scrub!"
697 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
698 [ "$STATUS" == "scanning" ] ||
699 error "(6) Expect 'scanning', but got '$STATUS'"
701 echo "stop $SINGLEMDS"
702 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
704 echo "start $SINGLEMDS with disabling OI scrub"
705 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
706 error "(8) Fail to start MDS!"
708 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
709 [ "$STATUS" == "paused" ] ||
710 error "(9) Expect 'paused', but got '$STATUS'"
712 echo "stop $SINGLEMDS"
713 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
715 echo "start $SINGLEMDS without disabling OI scrub"
716 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
717 error "(11) Fail to start MDS!"
719 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
720 [ "$STATUS" == "scanning" ] ||
721 error "(12) Expect 'scanning', but got '$STATUS'"
723 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
724 do_facet $SINGLEMDS $LCTL set_param fail_val=0
726 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
727 [ "$STATUS" == "completed" ] ||
728 error "(13) Expect 'completed', but got '$STATUS'"
730 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
731 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
733 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
739 formatall > /dev/null
744 local tname=`date +%s`
745 rm -rf $MOUNT/$tname > /dev/null
746 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
748 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
750 # reset OI scrub start point by force
751 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
753 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
754 [ "$STATUS" == "completed" ] ||
755 error "(4) Expect 'completed', but got '$STATUS'"
757 # OI scrub should skip the new created objects for the first accessing
758 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
759 # notice we're creating a new llog for every OST on every startup
760 # new features can make this even less stable, so we only check
761 # that the number of skipped files is less than 2x the number of files
762 local MAXIMUM=$((CREATED * 2))
763 local MINIMUM=$((CREATED + 1)) # files + directory
764 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
765 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
767 # reset OI scrub start point by force
768 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
770 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
771 [ "$STATUS" == "completed" ] ||
772 error "(7) Expect 'completed', but got '$STATUS'"
774 # OI scrub should skip the new created object only once
775 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
776 [ $SKIPPED -eq 0 ] ||
777 error "(8) Expect 0 objects skipped, but got $SKIPPED"
779 rm -rf $MOUNT/$tname > /dev/null
781 run_test 11 "OI scrub skips the new created objects only once"
783 # restore MDS/OST size
784 MDSSIZE=${SAVED_MDSSIZE}
785 OSTSIZE=${SAVED_OSTSIZE}
787 # cleanup the system at last