3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ "${MDSFSTYPE:-$FSTYPE}" != "ldiskfs" ] &&
21 skip "test OI scrub only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 SAVED_MDSSIZE=${MDSSIZE}
25 SAVED_OSTSIZE=${OSTSIZE}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
31 check_and_setup_lustre
34 MDT_DEV="${FSNAME}-MDT0000"
35 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
36 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
37 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
38 SHOW_SCRUB="do_facet $SINGLEMDS \
39 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
40 MOUNT_OPTS_SCRUB="-o user_xattr"
41 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
51 echo "preparing... ${nfiles} files will be created."
53 cp $LUSTRE/tests/*.sh $DIR/$tdir/
54 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
55 error "createmany failed"; }
58 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
59 echo "stop $SINGLEMDS"
60 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
65 echo "start $SINGLEMDS without disabling OI scrub"
66 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
67 error "(1) Fail to start MDS!"
69 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
70 [ "$STATUS" == "init" ] ||
71 error "(2) Expect 'init', but got '$STATUS'"
73 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
74 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
76 mount_client $MOUNT || error "(4) Fail to start client!"
78 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
79 error "(5) File diff failed unexpected!"
81 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
85 mds_remove_ois || error "(1) Fail to remove/recreate!"
87 echo "start $SINGLEMDS without disabling OI scrub"
88 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
89 error "(2) Fail to start MDS!"
92 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
93 [ "$STATUS" == "completed" ] ||
94 error "(3) Expect 'completed', but got '$STATUS'"
96 mount_client $MOUNT || error "(4) Fail to start client!"
98 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
99 error "(5) File diff failed unexpected!"
101 run_test 1a "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
106 # OI files to be removed:
109 # idx 2: oi.16.{2,4,8,16,32}
110 # idx 3: oi.16.{3,9,27}
111 # idx 5: oi.16.{5,25}
112 # idx 7: oi.16.{7,49}
113 for index in 0 1 2 3 5 7; do
115 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
117 echo "start $SINGLEMDS with disabling OI scrub"
118 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
119 /dev/null || error "(2) Fail to start MDS!"
121 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
122 [ "$FLAGS" == "recreated" ] ||
123 error "(3) Expect 'recreated', but got '$STATUS'"
125 $START_SCRUB || error "(4) Fail to start OI scrub!"
127 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
128 [ "$STATUS" == "completed" ] ||
129 error "(5) Expect 'completed', but got '$STATUS'"
131 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
133 error "(6) Expect empty flags, but got '$FLAGS'"
136 run_test 1b "Auto detect kinds of OI file(s) removed/recreated cases"
140 mds_backup_restore || error "(1) Fail to backup/restore!"
142 echo "start $SINGLEMDS without disabling OI scrub"
143 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
144 error "(2) Fail to start MDS!"
147 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
148 [ "$STATUS" == "completed" ] ||
149 error "(3) Expect 'completed', but got '$STATUS'"
151 mount_client $MOUNT || error "(4) Fail to start client!"
153 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
154 error "(5) File diff failed unexpected!"
156 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
160 mds_backup_restore || error "(1) Fail to backup/restore!"
162 echo "start $SINGLEMDS with disabling OI scrub"
163 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
164 error "(2) Fail to start MDS!"
166 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
167 [ "$STATUS" == "init" ] ||
168 error "(3) Expect 'init', but got '$STATUS'"
170 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
171 [ "$FLAGS" == "inconsistent" ] ||
172 error "(4) Expect 'inconsistent', but got '$FLAGS'"
176 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
180 mds_backup_restore || error "(1) Fail to backup/restore!"
182 echo "start $SINGLEMDS with disabling OI scrub"
183 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
184 error "(2) Fail to start MDS!"
186 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
187 [ "$STATUS" == "init" ] ||
188 error "(3) Expect 'init', but got '$STATUS'"
190 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
191 [ "$FLAGS" == "inconsistent" ] ||
192 error "(4) Expect 'inconsistent', but got '$FLAGS'"
194 mount_client $MOUNT || error "(5) Fail to start client!"
196 do_facet $SINGLEMDS \
197 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
198 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
199 error "(6) File diff failed unexpected!"
202 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
203 [ "$STATUS" == "completed" ] ||
204 error "(7) Expect 'completed', but got '$STATUS'"
206 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
210 mds_backup_restore || error "(1) Fail to backup/restore!"
212 echo "start $SINGLEMDS with disabling OI scrub"
213 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
214 error "(2) Fail to start MDS!"
216 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
217 [ "$STATUS" == "init" ] ||
218 error "(3) Expect 'init', but got '$STATUS'"
220 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
221 [ "$FLAGS" == "inconsistent" ] ||
222 error "(4) Expect 'inconsistent', but got '$FLAGS'"
224 mount_client $MOUNT || error "(5) Fail to start client!"
226 do_facet $SINGLEMDS \
227 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
228 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
229 do_facet $SINGLEMDS $LCTL set_param fail_val=3
230 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
231 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
232 error "(6) File diff failed unexpected!"
234 umount_client $MOUNT || error "(7) Fail to stop client!"
236 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
237 [ "$STATUS" == "scanning" ] ||
238 error "(8) Expect 'scanning', but got '$STATUS'"
240 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
241 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
243 echo "stop $SINGLEMDS"
244 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
246 echo "start $SINGLEMDS with disabling OI scrub"
247 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
248 error "(10) Fail to start MDS!"
250 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
251 [ "$STATUS" == "crashed" ] ||
252 error "(11) Expect 'crashed', but got '$STATUS'"
254 echo "stop $SINGLEMDS"
255 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
257 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
258 do_facet $SINGLEMDS $LCTL set_param fail_val=3
259 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
260 echo "start $SINGLEMDS without disabling OI scrub"
261 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
262 error "(13) Fail to start MDS!"
264 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
265 [ "$STATUS" == "scanning" ] ||
266 error "(14) Expect 'scanning', but got '$STATUS'"
268 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
269 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
271 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
272 [ "$STATUS" == "failed" ] ||
273 error "(15) Expect 'failed', but got '$STATUS'"
275 mount_client $MOUNT || error "(16) Fail to start client!"
277 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
278 do_facet $SINGLEMDS $LCTL set_param fail_val=3
279 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
280 stat $DIR/$tdir/${tfile}1000 ||
281 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
283 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
284 [ "$STATUS" == "scanning" ] ||
285 error "(18) Expect 'scanning', but got '$STATUS'"
287 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
288 do_facet $SINGLEMDS $LCTL set_param fail_val=0
290 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
291 [ "$STATUS" == "completed" ] ||
292 error "(19) Expect 'completed', but got '$STATUS'"
294 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
295 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
297 run_test 5 "OI scrub state machine"
301 mds_backup_restore || error "(1) Fail to backup/restore!"
303 echo "start $SINGLEMDS with disabling OI scrub"
304 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
305 error "(2) Fail to start MDS!"
307 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
308 [ "$STATUS" == "init" ] ||
309 error "(3) Expect 'init', but got '$STATUS'"
311 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
312 [ "$FLAGS" == "inconsistent" ] ||
313 error "(4) Expect 'inconsistent', but got '$FLAGS'"
315 mount_client $MOUNT || error "(5) Fail to start client!"
317 do_facet $SINGLEMDS \
318 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
319 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
320 do_facet $SINGLEMDS $LCTL set_param fail_val=3
321 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
322 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
323 error "(6) File diff failed unexpected!"
325 # Sleep 5 sec to guarantee at least one object processed by OI scrub
327 # Fail the OI scrub to guarantee there is at least one checkpoint
328 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
331 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
332 [ "$STATUS" == "failed" ] ||
333 error "(7) Expect 'failed', but got '$STATUS'"
335 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
336 do_facet $SINGLEMDS $LCTL set_param fail_val=3
337 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
338 # stat will re-trigger OI scrub
339 stat $DIR/$tdir/${tfile}800 ||
340 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
342 umount_client $MOUNT || error "(9) Fail to stop client!"
344 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
345 [ "$STATUS" == "scanning" ] ||
346 error "(10) Expect 'scanning', but got '$STATUS'"
348 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
349 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
351 local POSITION0=$($SHOW_SCRUB |
352 awk '/^last_checkpoint_position/ {print $2}')
353 POSITION0=$((POSITION0 + 1))
355 echo "stop $SINGLEMDS"
356 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
358 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
359 do_facet $SINGLEMDS $LCTL set_param fail_val=3
360 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
361 echo "start $SINGLEMDS without disabling OI scrub"
362 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
363 error "(12) Fail to start MDS!"
365 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
366 [ "$STATUS" == "scanning" ] ||
367 error "(13) Expect 'scanning', but got '$STATUS'"
369 local POSITION1=$($SHOW_SCRUB |
370 awk '/^latest_start_position/ {print $2}')
371 [ $POSITION0 -eq $POSITION1 ] ||
372 error "(14) Expect position: $POSITION0, but got $POSITION1"
374 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
375 do_facet $SINGLEMDS $LCTL set_param fail_val=0
377 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
378 [ "$STATUS" == "completed" ] ||
379 error "(15) Expect 'completed', but got '$STATUS'"
381 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
382 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
384 run_test 6 "OI scrub resumes from last checkpoint"
388 mds_backup_restore || error "(1) Fail to backup/restore!"
390 echo "start $SINGLEMDS with disabling OI scrub"
391 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
392 error "(2) Fail to start MDS!"
394 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
395 [ "$STATUS" == "init" ] ||
396 error "(3) Expect 'init', but got '$STATUS'"
398 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
399 [ "$FLAGS" == "inconsistent" ] ||
400 error "(4) Expect 'inconsistent', but got '$FLAGS'"
402 mount_client $MOUNT || error "(5) Fail to start client!"
404 do_facet $SINGLEMDS \
405 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
406 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
407 do_facet $SINGLEMDS $LCTL set_param fail_val=3
408 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
409 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
410 error "(6) File diff failed unexpected!"
412 stat $DIR/$tdir/${tfile}300 ||
413 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
415 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
416 [ "$STATUS" == "scanning" ] ||
417 error "(8) Expect 'scanning', but got '$STATUS'"
419 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
420 [ "$FLAGS" == "inconsistent,auto" ] ||
421 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
423 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
424 do_facet $SINGLEMDS $LCTL set_param fail_val=0
426 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
427 [ "$STATUS" == "completed" ] ||
428 error "(10) Expect 'completed', but got '$STATUS'"
430 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
431 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
433 run_test 7 "System is available during OI scrub scanning"
437 mds_backup_restore || error "(1) Fail to backup/restore!"
439 echo "start $SINGLEMDS with disabling OI scrub"
440 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
441 error "(2) Fail to start MDS!"
443 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
444 [ "$STATUS" == "init" ] ||
445 error "(3) Expect 'init', but got '$STATUS'"
447 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
448 [ "$FLAGS" == "inconsistent" ] ||
449 error "(4) Expect 'inconsistent', but got '$FLAGS'"
451 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
452 do_facet $SINGLEMDS $LCTL set_param fail_val=3
453 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
454 $START_SCRUB || error "(5) Fail to start OI scrub!"
456 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
457 [ "$STATUS" == "scanning" ] ||
458 error "(6) Expect 'scanning', but got '$STATUS'"
460 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
462 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
463 [ "$STATUS" == "stopped" ] ||
464 error "(8) Expect 'stopped', but got '$STATUS'"
466 $START_SCRUB || error "(9) Fail to start OI scrub!"
468 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "scanning" ] ||
470 error "(10) Expect 'scanning', but got '$STATUS'"
472 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
473 do_facet $SINGLEMDS $LCTL set_param fail_val=0
475 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
476 [ "$STATUS" == "completed" ] ||
477 error "(11) Expect 'completed', but got '$STATUS'"
479 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
480 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
482 run_test 8 "Control OI scrub manually"
485 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
486 skip "Testing on UP system, the speed may be inaccurate."
491 mds_backup_restore || error "(1) Fail to backup/restore!"
493 echo "start $SINGLEMDS with disabling OI scrub"
494 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
495 error "(2) Fail to start MDS!"
497 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
498 [ "$STATUS" == "init" ] ||
499 error "(3) Expect 'init', but got '$STATUS'"
501 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
502 [ "$FLAGS" == "inconsistent" ] ||
503 error "(4) Expect 'inconsistent', but got '$FLAGS'"
505 # OI scrub should run with full speed under inconsistent case
506 $START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
509 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
510 [ "$STATUS" == "completed" ] ||
511 error "(6) Expect 'completed', but got '$STATUS'"
513 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
514 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
516 # OI scrub should run with limited speed under non-inconsistent case
517 $START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
520 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
521 [ "$STATUS" == "scanning" ] ||
522 error "(9) Expect 'scanning', but got '$STATUS'"
524 # Do NOT ignore that there are 1024 pre-fetched items.
525 # So the max speed may be (1024 + 100 * 10) / 10.
526 # And there may be time error, so the max speed may be more large.
527 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
528 [ $SPEED -gt 220 ] &&
529 error "(10) Unexpected speed $SPEED, should not more than 220"
532 do_facet $SINGLEMDS \
533 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
536 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
537 [ $SPEED -lt 220 ] &&
538 error "(11) Unexpected speed $SPEED, should not less than 220"
540 [ $SPEED -gt 300 ] &&
541 error "(12) Unexpected speed $SPEED, should not more than 300"
543 do_facet $SINGLEMDS \
544 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
546 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
547 [ "$STATUS" == "completed" ] ||
548 error "(13) Expect 'completed', but got '$STATUS'"
550 run_test 9 "OI scrub speed control"
554 mds_backup_restore || error "(1) Fail to backup/restore!"
556 echo "start $SINGLEMDS with disabling OI scrub"
557 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
558 error "(2) Fail to start MDS!"
560 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
561 [ "$STATUS" == "init" ] ||
562 error "(3) Expect 'init', but got '$STATUS'"
564 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
565 [ "$FLAGS" == "inconsistent" ] ||
566 error "(4) Expect 'inconsistent', but got '$FLAGS'"
568 mount_client $MOUNT || error "(5) Fail to start client!"
570 do_facet $SINGLEMDS \
571 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
572 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
573 do_facet $SINGLEMDS $LCTL set_param fail_val=3
574 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
575 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
576 error "(6) File diff failed unexpected!"
578 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
579 [ "$STATUS" == "scanning" ] ||
580 error "(7) Expect 'scanning', but got '$STATUS'"
582 umount_client $MOUNT || error "(8) Fail to stop client!"
584 echo "stop $SINGLEMDS"
585 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
587 echo "start $SINGLEMDS with disabling OI scrub"
588 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
589 error "(10) Fail to start MDS!"
591 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
592 [ "$STATUS" == "paused" ] ||
593 error "(11) Expect 'paused', but got '$STATUS'"
595 echo "stop $SINGLEMDS"
596 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
598 echo "start $SINGLEMDS without disabling OI scrub"
599 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
600 error "(13) Fail to start MDS!"
602 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
603 [ "$STATUS" == "scanning" ] ||
604 error "(14) Expect 'scanning', but got '$STATUS'"
606 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
607 do_facet $SINGLEMDS $LCTL set_param fail_val=0
609 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
610 [ "$STATUS" == "completed" ] ||
611 error "(15) Expect 'completed', but got '$STATUS'"
613 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
614 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
616 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
620 mds_backup_restore || error "(1) Fail to backup/restore!"
622 echo "start $SINGLEMDS with disabling OI scrub"
623 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
624 error "(2) Fail to start MDS!"
626 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
627 [ "$STATUS" == "init" ] ||
628 error "(3) Expect 'init', but got '$STATUS'"
630 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
631 [ "$FLAGS" == "inconsistent" ] ||
632 error "(4) Expect 'inconsistent', but got '$FLAGS'"
634 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
635 do_facet $SINGLEMDS $LCTL set_param fail_val=3
636 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
638 $START_SCRUB || error "(5) Fail to start OI scrub!"
640 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
641 [ "$STATUS" == "scanning" ] ||
642 error "(6) Expect 'scanning', but got '$STATUS'"
644 echo "stop $SINGLEMDS"
645 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
647 echo "start $SINGLEMDS with disabling OI scrub"
648 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
649 error "(8) Fail to start MDS!"
651 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
652 [ "$STATUS" == "paused" ] ||
653 error "(9) Expect 'paused', but got '$STATUS'"
655 echo "stop $SINGLEMDS"
656 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
658 echo "start $SINGLEMDS without disabling OI scrub"
659 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
660 error "(11) Fail to start MDS!"
662 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
663 [ "$STATUS" == "scanning" ] ||
664 error "(12) Expect 'scanning', but got '$STATUS'"
666 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
667 do_facet $SINGLEMDS $LCTL set_param fail_val=0
669 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
670 [ "$STATUS" == "completed" ] ||
671 error "(13) Expect 'completed', but got '$STATUS'"
673 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
674 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
676 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
678 # restore the ${facet}_MKFS_OPTS variables
679 for facet in MGS MDS OST; do
680 opts=SAVED_${facet}_MKFS_OPTS
681 if [[ -n ${!opts} ]]; then
682 eval ${facet}_MKFS_OPTS=\"${!opts}\"
686 # restore MDS/OST size
687 MDSSIZE=${SAVED_MDSSIZE}
688 OSTSIZE=${SAVED_OSTSIZE}
690 # cleanup the system at last
693 complete $(basename $0) $SECONDS