3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
21 skip "test OI scrub only for ldiskfs" && exit 0
22 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
23 skip "Need MDS version at least 2.2.90" && exit 0
24 require_dsh_mds || exit 0
26 SAVED_MDSSIZE=${MDSSIZE}
27 SAVED_OSTSIZE=${OSTSIZE}
28 # use small MDS + OST size to speed formatting time
29 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
33 check_and_setup_lustre
36 MDT_DEV="${FSNAME}-MDT0000"
37 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
38 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
39 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
40 SHOW_SCRUB="do_facet $SINGLEMDS \
41 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
42 MOUNT_OPTS_SCRUB="-o user_xattr"
43 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
53 echo "preparing... ${nfiles} files will be created."
55 cp $LUSTRE/tests/*.sh $DIR/$tdir/
56 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
57 error "createmany failed"; }
60 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
61 echo "stop $SINGLEMDS"
62 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
67 echo "start $SINGLEMDS without disabling OI scrub"
68 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
69 error "(1) Fail to start MDS!"
71 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
72 [ "$STATUS" == "init" ] ||
73 error "(2) Expect 'init', but got '$STATUS'"
75 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
76 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
78 mount_client $MOUNT || error "(4) Fail to start client!"
80 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
81 error "(5) File diff failed unexpected!"
83 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
87 mds_remove_ois || error "(1) Fail to remove/recreate!"
89 echo "start $SINGLEMDS without disabling OI scrub"
90 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
91 error "(2) Fail to start MDS!"
94 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
95 [ "$STATUS" == "completed" ] ||
96 error "(3) Expect 'completed', but got '$STATUS'"
98 mount_client $MOUNT || error "(4) Fail to start client!"
100 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
101 error "(5) File diff failed unexpected!"
103 run_test 1a "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
108 # OI files to be removed:
111 # idx 2: oi.16.{2,4,8,16,32}
112 # idx 3: oi.16.{3,9,27}
113 # idx 5: oi.16.{5,25}
114 # idx 7: oi.16.{7,49}
115 for index in 0 1 2 3 5 7; do
117 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
119 echo "start $SINGLEMDS with disabling OI scrub"
120 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
121 /dev/null || error "(2) Fail to start MDS!"
123 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
124 [ "$FLAGS" == "recreated" ] ||
125 error "(3) Expect 'recreated', but got '$FLAGS'"
127 $START_SCRUB || error "(4) Fail to start OI scrub!"
129 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
130 [ "$STATUS" == "completed" ] ||
131 error "(5) Expect 'completed', but got '$STATUS'"
133 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
135 error "(6) Expect empty flags, but got '$FLAGS'"
138 run_test 1b "Auto detect kinds of OI file(s) removed/recreated cases"
142 mds_backup_restore || error "(1) Fail to backup/restore!"
144 echo "start $SINGLEMDS without disabling OI scrub"
145 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
146 error "(2) Fail to start MDS!"
149 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
150 [ "$STATUS" == "completed" ] ||
151 error "(3) Expect 'completed', but got '$STATUS'"
153 mount_client $MOUNT || error "(4) Fail to start client!"
155 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
156 error "(5) File diff failed unexpected!"
158 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
162 mds_backup_restore || error "(1) Fail to backup/restore!"
164 echo "start $SINGLEMDS with disabling OI scrub"
165 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
166 error "(2) Fail to start MDS!"
168 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
169 [ "$STATUS" == "init" ] ||
170 error "(3) Expect 'init', but got '$STATUS'"
172 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
173 [ "$FLAGS" == "inconsistent" ] ||
174 error "(4) Expect 'inconsistent', but got '$FLAGS'"
178 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
182 mds_backup_restore || error "(1) Fail to backup/restore!"
184 echo "start $SINGLEMDS with disabling OI scrub"
185 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
186 error "(2) Fail to start MDS!"
188 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
189 [ "$STATUS" == "init" ] ||
190 error "(3) Expect 'init', but got '$STATUS'"
192 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
193 [ "$FLAGS" == "inconsistent" ] ||
194 error "(4) Expect 'inconsistent', but got '$FLAGS'"
196 mount_client $MOUNT || error "(5) Fail to start client!"
198 do_facet $SINGLEMDS \
199 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
200 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
201 error "(6) File diff failed unexpected!"
204 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
205 [ "$STATUS" == "completed" ] ||
206 error "(7) Expect 'completed', but got '$STATUS'"
208 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
212 mds_backup_restore || error "(1) Fail to backup/restore!"
214 echo "start $SINGLEMDS with disabling OI scrub"
215 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
216 error "(2) Fail to start MDS!"
218 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
219 [ "$STATUS" == "init" ] ||
220 error "(3) Expect 'init', but got '$STATUS'"
222 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
223 [ "$FLAGS" == "inconsistent" ] ||
224 error "(4) Expect 'inconsistent', but got '$FLAGS'"
226 mount_client $MOUNT || error "(5) Fail to start client!"
228 do_facet $SINGLEMDS \
229 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
230 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
231 do_facet $SINGLEMDS $LCTL set_param fail_val=3
232 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
233 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
234 error "(6) File diff failed unexpected!"
236 umount_client $MOUNT || error "(7) Fail to stop client!"
238 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
239 [ "$STATUS" == "scanning" ] ||
240 error "(8) Expect 'scanning', but got '$STATUS'"
242 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
243 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
245 echo "stop $SINGLEMDS"
246 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
248 echo "start $SINGLEMDS with disabling OI scrub"
249 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
250 error "(10) Fail to start MDS!"
252 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
253 [ "$STATUS" == "crashed" ] ||
254 error "(11) Expect 'crashed', but got '$STATUS'"
256 echo "stop $SINGLEMDS"
257 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
259 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
260 do_facet $SINGLEMDS $LCTL set_param fail_val=3
261 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
262 echo "start $SINGLEMDS without disabling OI scrub"
263 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
264 error "(13) Fail to start MDS!"
266 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
267 [ "$STATUS" == "scanning" ] ||
268 error "(14) Expect 'scanning', but got '$STATUS'"
270 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
271 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
273 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
274 [ "$STATUS" == "failed" ] ||
275 error "(15) Expect 'failed', but got '$STATUS'"
277 mount_client $MOUNT || error "(16) Fail to start client!"
279 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
280 do_facet $SINGLEMDS $LCTL set_param fail_val=3
281 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
282 stat $DIR/$tdir/${tfile}1000 ||
283 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
285 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
286 [ "$STATUS" == "scanning" ] ||
287 error "(18) Expect 'scanning', but got '$STATUS'"
289 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
290 do_facet $SINGLEMDS $LCTL set_param fail_val=0
292 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
293 [ "$STATUS" == "completed" ] ||
294 error "(19) Expect 'completed', but got '$STATUS'"
296 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
297 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
299 run_test 5 "OI scrub state machine"
303 mds_backup_restore || error "(1) Fail to backup/restore!"
305 echo "start $SINGLEMDS with disabling OI scrub"
306 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
307 error "(2) Fail to start MDS!"
309 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
310 [ "$STATUS" == "init" ] ||
311 error "(3) Expect 'init', but got '$STATUS'"
313 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
314 [ "$FLAGS" == "inconsistent" ] ||
315 error "(4) Expect 'inconsistent', but got '$FLAGS'"
317 mount_client $MOUNT || error "(5) Fail to start client!"
319 do_facet $SINGLEMDS \
320 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
321 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
322 do_facet $SINGLEMDS $LCTL set_param fail_val=3
323 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
324 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
325 error "(6) File diff failed unexpected!"
327 # Sleep 5 sec to guarantee at least one object processed by OI scrub
329 # Fail the OI scrub to guarantee there is at least one checkpoint
330 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
331 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
333 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
334 [ "$STATUS" == "failed" ] ||
335 error "(7) Expect 'failed', but got '$STATUS'"
337 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
338 do_facet $SINGLEMDS $LCTL set_param fail_val=3
339 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
340 # stat will re-trigger OI scrub
341 stat $DIR/$tdir/${tfile}800 ||
342 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
344 umount_client $MOUNT || error "(9) Fail to stop client!"
346 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
347 [ "$STATUS" == "scanning" ] ||
348 error "(10) Expect 'scanning', but got '$STATUS'"
350 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
351 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
353 local POSITION0=$($SHOW_SCRUB |
354 awk '/^last_checkpoint_position/ {print $2}')
355 POSITION0=$((POSITION0 + 1))
357 echo "stop $SINGLEMDS"
358 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
360 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
361 do_facet $SINGLEMDS $LCTL set_param fail_val=3
362 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
363 echo "start $SINGLEMDS without disabling OI scrub"
364 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
365 error "(12) Fail to start MDS!"
367 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
368 [ "$STATUS" == "scanning" ] ||
369 error "(13) Expect 'scanning', but got '$STATUS'"
371 local POSITION1=$($SHOW_SCRUB |
372 awk '/^latest_start_position/ {print $2}')
373 [ $POSITION0 -eq $POSITION1 ] ||
374 error "(14) Expect position: $POSITION0, but got $POSITION1"
376 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
377 do_facet $SINGLEMDS $LCTL set_param fail_val=0
379 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
380 [ "$STATUS" == "completed" ] ||
381 error "(15) Expect 'completed', but got '$STATUS'"
383 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
384 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
386 run_test 6 "OI scrub resumes from last checkpoint"
390 mds_backup_restore || error "(1) Fail to backup/restore!"
392 echo "start $SINGLEMDS with disabling OI scrub"
393 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
394 error "(2) Fail to start MDS!"
396 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
397 [ "$STATUS" == "init" ] ||
398 error "(3) Expect 'init', but got '$STATUS'"
400 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
401 [ "$FLAGS" == "inconsistent" ] ||
402 error "(4) Expect 'inconsistent', but got '$FLAGS'"
404 mount_client $MOUNT || error "(5) Fail to start client!"
406 do_facet $SINGLEMDS \
407 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
408 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
409 do_facet $SINGLEMDS $LCTL set_param fail_val=3
410 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
411 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
412 error "(6) File diff failed unexpected!"
414 stat $DIR/$tdir/${tfile}300 ||
415 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
417 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
418 [ "$STATUS" == "scanning" ] ||
419 error "(8) Expect 'scanning', but got '$STATUS'"
421 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
422 [ "$FLAGS" == "inconsistent,auto" ] ||
423 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
425 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
426 do_facet $SINGLEMDS $LCTL set_param fail_val=0
428 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
429 [ "$STATUS" == "completed" ] ||
430 error "(10) Expect 'completed', but got '$STATUS'"
432 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
433 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
435 run_test 7 "System is available during OI scrub scanning"
439 mds_backup_restore || error "(1) Fail to backup/restore!"
441 echo "start $SINGLEMDS with disabling OI scrub"
442 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
443 error "(2) Fail to start MDS!"
445 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
446 [ "$STATUS" == "init" ] ||
447 error "(3) Expect 'init', but got '$STATUS'"
449 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
450 [ "$FLAGS" == "inconsistent" ] ||
451 error "(4) Expect 'inconsistent', but got '$FLAGS'"
453 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
454 do_facet $SINGLEMDS $LCTL set_param fail_val=3
455 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
456 $START_SCRUB || error "(5) Fail to start OI scrub!"
458 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
459 [ "$STATUS" == "scanning" ] ||
460 error "(6) Expect 'scanning', but got '$STATUS'"
462 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
464 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
465 [ "$STATUS" == "stopped" ] ||
466 error "(8) Expect 'stopped', but got '$STATUS'"
468 $START_SCRUB || error "(9) Fail to start OI scrub!"
470 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
471 [ "$STATUS" == "scanning" ] ||
472 error "(10) Expect 'scanning', but got '$STATUS'"
474 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
475 do_facet $SINGLEMDS $LCTL set_param fail_val=0
477 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
478 [ "$STATUS" == "completed" ] ||
479 error "(11) Expect 'completed', but got '$STATUS'"
481 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
482 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
484 run_test 8 "Control OI scrub manually"
487 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
488 skip "Testing on UP system, the speed may be inaccurate."
493 mds_backup_restore || error "(1) Fail to backup/restore!"
495 echo "start $SINGLEMDS with disabling OI scrub"
496 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
497 error "(2) Fail to start MDS!"
499 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
500 [ "$STATUS" == "init" ] ||
501 error "(3) Expect 'init', but got '$STATUS'"
503 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
504 [ "$FLAGS" == "inconsistent" ] ||
505 error "(4) Expect 'inconsistent', but got '$FLAGS'"
507 # OI scrub should run with full speed under inconsistent case
508 $START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
511 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
512 [ "$STATUS" == "completed" ] ||
513 error "(6) Expect 'completed', but got '$STATUS'"
515 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
516 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
518 # OI scrub should run with limited speed under non-inconsistent case
519 $START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
522 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
523 [ "$STATUS" == "scanning" ] ||
524 error "(9) Expect 'scanning', but got '$STATUS'"
526 # Do NOT ignore that there are 1024 pre-fetched items.
527 # So the max speed may be (1024 + 100 * 10) / 10.
528 # And there may be time error, so the max speed may be more large.
529 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
530 [ $SPEED -gt 220 ] &&
531 error "(10) Unexpected speed $SPEED, should not more than 220"
534 do_facet $SINGLEMDS \
535 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
538 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
539 [ $SPEED -lt 220 ] &&
540 error "(11) Unexpected speed $SPEED, should not less than 220"
542 [ $SPEED -gt 300 ] &&
543 error "(12) Unexpected speed $SPEED, should not more than 300"
545 do_facet $SINGLEMDS \
546 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
548 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
549 [ "$STATUS" == "completed" ] ||
550 error "(13) Expect 'completed', but got '$STATUS'"
552 run_test 9 "OI scrub speed control"
556 mds_backup_restore || error "(1) Fail to backup/restore!"
558 echo "start $SINGLEMDS with disabling OI scrub"
559 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
560 error "(2) Fail to start MDS!"
562 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
563 [ "$STATUS" == "init" ] ||
564 error "(3) Expect 'init', but got '$STATUS'"
566 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
567 [ "$FLAGS" == "inconsistent" ] ||
568 error "(4) Expect 'inconsistent', but got '$FLAGS'"
570 mount_client $MOUNT || error "(5) Fail to start client!"
572 do_facet $SINGLEMDS \
573 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
574 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
575 do_facet $SINGLEMDS $LCTL set_param fail_val=3
576 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
577 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
578 error "(6) File diff failed unexpected!"
580 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
581 [ "$STATUS" == "scanning" ] ||
582 error "(7) Expect 'scanning', but got '$STATUS'"
584 umount_client $MOUNT || error "(8) Fail to stop client!"
586 echo "stop $SINGLEMDS"
587 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
589 echo "start $SINGLEMDS with disabling OI scrub"
590 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
591 error "(10) Fail to start MDS!"
593 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
594 [ "$STATUS" == "paused" ] ||
595 error "(11) Expect 'paused', but got '$STATUS'"
597 echo "stop $SINGLEMDS"
598 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
600 echo "start $SINGLEMDS without disabling OI scrub"
601 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
602 error "(13) Fail to start MDS!"
604 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
605 [ "$STATUS" == "scanning" ] ||
606 error "(14) Expect 'scanning', but got '$STATUS'"
608 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
609 do_facet $SINGLEMDS $LCTL set_param fail_val=0
611 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
612 [ "$STATUS" == "completed" ] ||
613 error "(15) Expect 'completed', but got '$STATUS'"
615 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
616 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
618 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
622 mds_backup_restore || error "(1) Fail to backup/restore!"
624 echo "start $SINGLEMDS with disabling OI scrub"
625 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
626 error "(2) Fail to start MDS!"
628 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
629 [ "$STATUS" == "init" ] ||
630 error "(3) Expect 'init', but got '$STATUS'"
632 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
633 [ "$FLAGS" == "inconsistent" ] ||
634 error "(4) Expect 'inconsistent', but got '$FLAGS'"
636 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
637 do_facet $SINGLEMDS $LCTL set_param fail_val=3
638 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
640 $START_SCRUB || error "(5) Fail to start OI scrub!"
642 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
643 [ "$STATUS" == "scanning" ] ||
644 error "(6) Expect 'scanning', but got '$STATUS'"
646 echo "stop $SINGLEMDS"
647 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
649 echo "start $SINGLEMDS with disabling OI scrub"
650 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
651 error "(8) Fail to start MDS!"
653 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
654 [ "$STATUS" == "paused" ] ||
655 error "(9) Expect 'paused', but got '$STATUS'"
657 echo "stop $SINGLEMDS"
658 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
660 echo "start $SINGLEMDS without disabling OI scrub"
661 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
662 error "(11) Fail to start MDS!"
664 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
665 [ "$STATUS" == "scanning" ] ||
666 error "(12) Expect 'scanning', but got '$STATUS'"
668 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
669 do_facet $SINGLEMDS $LCTL set_param fail_val=0
671 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
672 [ "$STATUS" == "completed" ] ||
673 error "(13) Expect 'completed', but got '$STATUS'"
675 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
676 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
678 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
687 local tname=`date +%s`
688 rm -rf $MOUNT/$tname > /dev/null
689 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
691 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
693 # reset OI scrub start point by force
694 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
696 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
697 [ "$STATUS" == "completed" ] ||
698 error "(4) Expect 'completed', but got '$STATUS'"
700 # OI scrub should skip the new created objects for the first accessing
701 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
702 # notice we're creating a new llog for every OST on every startup
703 # new features can make this even less stable, so we only check
704 # that the number of skipped files is less than 1.5x the number of files
705 local MAXIMUM=$((CREATED * 3 / 2))
706 local MINIMUM=$((CREATED + 1)) # files + directory
707 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM] &&
708 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
710 # reset OI scrub start point by force
711 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
713 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
714 [ "$STATUS" == "completed" ] ||
715 error "(7) Expect 'completed', but got '$STATUS'"
717 # OI scrub should skip the new created object only once
718 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
719 [ $SKIPPED -eq 0 ] ||
720 error "(8) Expect 0 objects skipped, but got $SKIPPED"
722 rm -rf $MOUNT/$tname > /dev/null
724 run_test 11 "OI scrub skips the new created objects only once"
726 # restore the ${facet}_MKFS_OPTS variables
727 for facet in MGS MDS OST; do
728 opts=SAVED_${facet}_MKFS_OPTS
729 if [[ -n ${!opts} ]]; then
730 eval ${facet}_MKFS_OPTS=\"${!opts}\"
734 # restore MDS/OST size
735 MDSSIZE=${SAVED_MDSSIZE}
736 OSTSIZE=${SAVED_OSTSIZE}
738 # cleanup the system at last