3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ "${MDSFSTYPE:-$FSTYPE}" != "ldiskfs" ] &&
21 skip "test OI scrub only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 SAVED_MDSSIZE=${MDSSIZE}
25 SAVED_OSTSIZE=${OSTSIZE}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
31 check_and_setup_lustre
34 MDT_DEV="${FSNAME}-MDT0000"
35 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
36 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
37 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
38 SHOW_SCRUB="do_facet $SINGLEMDS \
39 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
40 MOUNT_OPTS_SCRUB="-o user_xattr"
41 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
51 echo "preparing... ${nfiles} files will be created."
53 cp $LUSTRE/tests/*.sh $DIR/$tdir/
54 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
55 error "createmany failed"; }
58 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
59 echo "stop $SINGLEMDS"
60 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
65 echo "start $SINGLEMDS without disabling OI scrub"
66 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
67 error "(1) Fail to start MDS!"
69 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
70 [ "$STATUS" == "init" ] ||
71 error "(2) Expect 'init', but got '$STATUS'"
73 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
74 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
76 mount_client $MOUNT || error "(4) Fail to start client!"
78 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
79 error "(5) File diff failed unexpected!"
81 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
85 mds_remove_ois || error "(1) Fail to remove/recreate!"
87 echo "start $SINGLEMDS without disabling OI scrub"
88 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
89 error "(2) Fail to start MDS!"
92 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
93 [ "$STATUS" == "completed" ] ||
94 error "(3) Expect 'completed', but got '$STATUS'"
96 mount_client $MOUNT || error "(4) Fail to start client!"
98 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
99 error "(5) File diff failed unexpected!"
101 run_test 1a "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
106 # OI files to be removed:
109 # idx 2: oi.16.{2,4,8,16,32}
110 # idx 3: oi.16.{3,9,27}
111 # idx 5: oi.16.{5,25}
112 # idx 7: oi.16.{7,49}
113 for index in 0 1 2 3 5 7; do
115 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
117 echo "start $SINGLEMDS with disabling OI scrub"
118 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
119 /dev/null || error "(2) Fail to start MDS!"
121 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
122 [ "$FLAGS" == "recreated" ] ||
123 error "(3) Expect 'recreated', but got '$STATUS'"
125 $START_SCRUB || error "(4) Fail to start OI scrub!"
127 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
128 [ "$STATUS" == "completed" ] ||
129 error "(5) Expect 'completed', but got '$STATUS'"
131 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
133 error "(6) Expect empty flags, but got '$FLAGS'"
136 run_test 1b "Auto detect kinds of OI file(s) removed/recreated cases"
140 mds_backup_restore || error "(1) Fail to backup/restore!"
142 echo "start $SINGLEMDS without disabling OI scrub"
143 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
144 error "(2) Fail to start MDS!"
147 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
148 [ "$STATUS" == "completed" ] ||
149 error "(3) Expect 'completed', but got '$STATUS'"
151 mount_client $MOUNT || error "(4) Fail to start client!"
153 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
154 error "(5) File diff failed unexpected!"
156 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
160 mds_backup_restore || error "(1) Fail to backup/restore!"
162 echo "start $SINGLEMDS with disabling OI scrub"
163 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
164 error "(2) Fail to start MDS!"
166 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
167 [ "$STATUS" == "init" ] ||
168 error "(3) Expect 'init', but got '$STATUS'"
170 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
171 [ "$FLAGS" == "inconsistent" ] ||
172 error "(4) Expect 'inconsistent', but got '$FLAGS'"
176 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
180 mds_backup_restore || error "(1) Fail to backup/restore!"
182 echo "start $SINGLEMDS with disabling OI scrub"
183 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
184 error "(2) Fail to start MDS!"
186 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
187 [ "$STATUS" == "init" ] ||
188 error "(3) Expect 'init', but got '$STATUS'"
190 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
191 [ "$FLAGS" == "inconsistent" ] ||
192 error "(4) Expect 'inconsistent', but got '$FLAGS'"
194 mount_client $MOUNT || error "(5) Fail to start client!"
196 do_facet $SINGLEMDS \
197 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
198 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
199 error "(6) File diff failed unexpected!"
202 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
203 [ "$STATUS" == "completed" ] ||
204 error "(7) Expect 'completed', but got '$STATUS'"
206 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
210 mds_backup_restore || error "(1) Fail to backup/restore!"
212 echo "start $SINGLEMDS with disabling OI scrub"
213 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
214 error "(2) Fail to start MDS!"
216 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
217 [ "$STATUS" == "init" ] ||
218 error "(3) Expect 'init', but got '$STATUS'"
220 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
221 [ "$FLAGS" == "inconsistent" ] ||
222 error "(4) Expect 'inconsistent', but got '$FLAGS'"
224 mount_client $MOUNT || error "(5) Fail to start client!"
226 do_facet $SINGLEMDS \
227 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
228 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
229 do_facet $SINGLEMDS $LCTL set_param fail_val=3
230 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
231 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
232 error "(6) File diff failed unexpected!"
234 umount_client $MOUNT || error "(7) Fail to stop client!"
236 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
237 [ "$STATUS" == "scanning" ] ||
238 error "(8) Expect 'scanning', but got '$STATUS'"
240 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
241 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
243 echo "stop $SINGLEMDS"
244 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
246 echo "start $SINGLEMDS with disabling OI scrub"
247 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
248 error "(10) Fail to start MDS!"
250 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
251 [ "$STATUS" == "crashed" ] ||
252 error "(11) Expect 'crashed', but got '$STATUS'"
254 echo "stop $SINGLEMDS"
255 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
257 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
258 do_facet $SINGLEMDS $LCTL set_param fail_val=3
259 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
260 echo "start $SINGLEMDS without disabling OI scrub"
261 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
262 error "(13) Fail to start MDS!"
264 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
265 [ "$STATUS" == "scanning" ] ||
266 error "(14) Expect 'scanning', but got '$STATUS'"
268 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
269 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
271 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
272 [ "$STATUS" == "failed" ] ||
273 error "(15) Expect 'failed', but got '$STATUS'"
275 mount_client $MOUNT || error "(16) Fail to start client!"
277 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
278 do_facet $SINGLEMDS $LCTL set_param fail_val=3
279 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
280 stat $DIR/$tdir/${tfile}1000 ||
281 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
283 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
284 [ "$STATUS" == "scanning" ] ||
285 error "(18) Expect 'scanning', but got '$STATUS'"
287 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
288 do_facet $SINGLEMDS $LCTL set_param fail_val=0
290 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
291 [ "$STATUS" == "completed" ] ||
292 error "(19) Expect 'completed', but got '$STATUS'"
294 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
295 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
297 run_test 5 "OI scrub state machine"
301 mds_backup_restore || error "(1) Fail to backup/restore!"
303 echo "start $SINGLEMDS with disabling OI scrub"
304 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
305 error "(2) Fail to start MDS!"
307 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
308 [ "$STATUS" == "init" ] ||
309 error "(3) Expect 'init', but got '$STATUS'"
311 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
312 [ "$FLAGS" == "inconsistent" ] ||
313 error "(4) Expect 'inconsistent', but got '$FLAGS'"
315 mount_client $MOUNT || error "(5) Fail to start client!"
317 do_facet $SINGLEMDS \
318 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
319 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
320 do_facet $SINGLEMDS $LCTL set_param fail_val=3
321 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
322 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
323 error "(6) File diff failed unexpected!"
325 # Sleep 5 sec to guarantee at least one object processed by OI scrub
327 # Fail the OI scrub to guarantee there is at least one checkpoint
328 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
331 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
332 [ "$STATUS" == "failed" ] ||
333 error "(7) Expect 'failed', but got '$STATUS'"
335 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
336 do_facet $SINGLEMDS $LCTL set_param fail_val=3
337 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
338 # stat will re-trigger OI scrub
339 stat $DIR/$tdir/${tfile}800 ||
340 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
342 umount_client $MOUNT || error "(9) Fail to stop client!"
344 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
345 [ "$STATUS" == "scanning" ] ||
346 error "(10) Expect 'scanning', but got '$STATUS'"
348 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
349 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
351 local POSITION0=$($SHOW_SCRUB |
352 awk '/^last_checkpoint_position/ {print $2}')
353 POSITION0=$((POSITION0 + 1))
355 echo "stop $SINGLEMDS"
356 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
358 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
359 do_facet $SINGLEMDS $LCTL set_param fail_val=3
360 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
361 echo "start $SINGLEMDS without disabling OI scrub"
362 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
363 error "(12) Fail to start MDS!"
365 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
366 [ "$STATUS" == "scanning" ] ||
367 error "(13) Expect 'scanning', but got '$STATUS'"
369 local POSITION1=$($SHOW_SCRUB |
370 awk '/^latest_start_position/ {print $2}')
371 [ $POSITION0 -eq $POSITION1 ] ||
372 error "(14) Expect position: $POSITION0, but got $POSITION1"
374 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
375 do_facet $SINGLEMDS $LCTL set_param fail_val=0
377 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
378 [ "$STATUS" == "completed" ] ||
379 error "(15) Expect 'completed', but got '$STATUS'"
381 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
382 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
384 run_test 6 "OI scrub resumes from last checkpoint"
388 mds_backup_restore || error "(1) Fail to backup/restore!"
390 echo "start $SINGLEMDS with disabling OI scrub"
391 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
392 error "(2) Fail to start MDS!"
394 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
395 [ "$STATUS" == "init" ] ||
396 error "(3) Expect 'init', but got '$STATUS'"
398 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
399 [ "$FLAGS" == "inconsistent" ] ||
400 error "(4) Expect 'inconsistent', but got '$FLAGS'"
402 mount_client $MOUNT || error "(5) Fail to start client!"
404 do_facet $SINGLEMDS \
405 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
406 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
407 do_facet $SINGLEMDS $LCTL set_param fail_val=3
408 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
409 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
410 error "(6) File diff failed unexpected!"
412 stat $DIR/$tdir/${tfile}300 ||
413 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
415 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
416 [ "$STATUS" == "scanning" ] ||
417 error "(8) Expect 'scanning', but got '$STATUS'"
419 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
420 [ "$FLAGS" == "inconsistent,auto" ] ||
421 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
423 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
424 do_facet $SINGLEMDS $LCTL set_param fail_val=0
426 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
427 [ "$STATUS" == "completed" ] ||
428 error "(10) Expect 'completed', but got '$STATUS'"
430 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
431 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
433 run_test 7 "System is available during OI scrub scanning"
437 mds_backup_restore || error "(1) Fail to backup/restore!"
439 echo "start $SINGLEMDS with disabling OI scrub"
440 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
441 error "(2) Fail to start MDS!"
443 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
444 [ "$STATUS" == "init" ] ||
445 error "(3) Expect 'init', but got '$STATUS'"
447 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
448 [ "$FLAGS" == "inconsistent" ] ||
449 error "(4) Expect 'inconsistent', but got '$FLAGS'"
451 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
452 do_facet $SINGLEMDS $LCTL set_param fail_val=3
453 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
454 $START_SCRUB || error "(5) Fail to start OI scrub!"
456 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
457 [ "$STATUS" == "scanning" ] ||
458 error "(6) Expect 'scanning', but got '$STATUS'"
460 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
462 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
463 [ "$STATUS" == "stopped" ] ||
464 error "(8) Expect 'stopped', but got '$STATUS'"
466 $START_SCRUB || error "(9) Fail to start OI scrub!"
468 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "scanning" ] ||
470 error "(10) Expect 'scanning', but got '$STATUS'"
472 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
473 do_facet $SINGLEMDS $LCTL set_param fail_val=0
475 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
476 [ "$STATUS" == "completed" ] ||
477 error "(11) Expect 'completed', but got '$STATUS'"
479 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
480 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
482 run_test 8 "Control OI scrub manually"
486 mds_backup_restore || error "(1) Fail to backup/restore!"
488 echo "start $SINGLEMDS with disabling OI scrub"
489 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
490 error "(2) Fail to start MDS!"
492 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
493 [ "$STATUS" == "init" ] ||
494 error "(3) Expect 'init', but got '$STATUS'"
496 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
497 [ "$FLAGS" == "inconsistent" ] ||
498 error "(4) Expect 'inconsistent', but got '$FLAGS'"
500 # OI scrub should run with full speed under inconsistent case
501 $START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
504 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
505 [ "$STATUS" == "completed" ] ||
506 error "(6) Expect 'completed', but got '$STATUS'"
508 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
509 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
511 # OI scrub should run with limited speed under non-inconsistent case
512 $START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
515 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
516 [ "$STATUS" == "scanning" ] ||
517 error "(9) Expect 'scanning', but got '$STATUS'"
519 # Do NOT ignore that there are 1024 pre-fetched items.
520 # So the max speed may be (1024 + 100 * 10) / 10.
521 # And there may be time error, so the max speed may be more large.
522 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
523 [ $SPEED -gt 220 ] &&
524 error "(10) Unexpected speed $SPEED, should not more than 220"
527 do_facet $SINGLEMDS \
528 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
531 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
532 [ $SPEED -lt 220 ] &&
533 error "(11) Unexpected speed $SPEED, should not less than 220"
535 [ $SPEED -gt 300 ] &&
536 error "(12) Unexpected speed $SPEED, should not more than 300"
538 do_facet $SINGLEMDS \
539 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
541 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
542 [ "$STATUS" == "completed" ] ||
543 error "(13) Expect 'completed', but got '$STATUS'"
545 run_test 9 "OI scrub speed control"
549 mds_backup_restore || error "(1) Fail to backup/restore!"
551 echo "start $SINGLEMDS with disabling OI scrub"
552 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
553 error "(2) Fail to start MDS!"
555 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
556 [ "$STATUS" == "init" ] ||
557 error "(3) Expect 'init', but got '$STATUS'"
559 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
560 [ "$FLAGS" == "inconsistent" ] ||
561 error "(4) Expect 'inconsistent', but got '$FLAGS'"
563 mount_client $MOUNT || error "(5) Fail to start client!"
565 do_facet $SINGLEMDS \
566 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
567 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
568 do_facet $SINGLEMDS $LCTL set_param fail_val=3
569 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
570 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
571 error "(6) File diff failed unexpected!"
573 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
574 [ "$STATUS" == "scanning" ] ||
575 error "(7) Expect 'scanning', but got '$STATUS'"
577 umount_client $MOUNT || error "(8) Fail to stop client!"
579 echo "stop $SINGLEMDS"
580 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
582 echo "start $SINGLEMDS with disabling OI scrub"
583 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
584 error "(10) Fail to start MDS!"
586 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
587 [ "$STATUS" == "paused" ] ||
588 error "(11) Expect 'paused', but got '$STATUS'"
590 echo "stop $SINGLEMDS"
591 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
593 echo "start $SINGLEMDS without disabling OI scrub"
594 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
595 error "(13) Fail to start MDS!"
597 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
598 [ "$STATUS" == "scanning" ] ||
599 error "(14) Expect 'scanning', but got '$STATUS'"
601 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
602 do_facet $SINGLEMDS $LCTL set_param fail_val=0
604 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
605 [ "$STATUS" == "completed" ] ||
606 error "(15) Expect 'completed', but got '$STATUS'"
608 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
609 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
611 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
615 mds_backup_restore || error "(1) Fail to backup/restore!"
617 echo "start $SINGLEMDS with disabling OI scrub"
618 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
619 error "(2) Fail to start MDS!"
621 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
622 [ "$STATUS" == "init" ] ||
623 error "(3) Expect 'init', but got '$STATUS'"
625 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
626 [ "$FLAGS" == "inconsistent" ] ||
627 error "(4) Expect 'inconsistent', but got '$FLAGS'"
629 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
630 do_facet $SINGLEMDS $LCTL set_param fail_val=3
631 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
633 $START_SCRUB || error "(5) Fail to start OI scrub!"
635 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
636 [ "$STATUS" == "scanning" ] ||
637 error "(6) Expect 'scanning', but got '$STATUS'"
639 echo "stop $SINGLEMDS"
640 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
642 echo "start $SINGLEMDS with disabling OI scrub"
643 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
644 error "(8) Fail to start MDS!"
646 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
647 [ "$STATUS" == "paused" ] ||
648 error "(9) Expect 'paused', but got '$STATUS'"
650 echo "stop $SINGLEMDS"
651 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
653 echo "start $SINGLEMDS without disabling OI scrub"
654 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
655 error "(11) Fail to start MDS!"
657 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
658 [ "$STATUS" == "scanning" ] ||
659 error "(12) Expect 'scanning', but got '$STATUS'"
661 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
662 do_facet $SINGLEMDS $LCTL set_param fail_val=0
664 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
665 [ "$STATUS" == "completed" ] ||
666 error "(13) Expect 'completed', but got '$STATUS'"
668 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
669 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
671 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
673 # restore the ${facet}_MKFS_OPTS variables
674 for facet in MGS MDS OST; do
675 opts=SAVED_${facet}_MKFS_OPTS
676 if [[ -n ${!opts} ]]; then
677 eval ${facet}_MKFS_OPTS=\"${!opts}\"
681 # restore MDS/OST size
682 MDSSIZE=${SAVED_MDSSIZE}
683 OSTSIZE=${SAVED_OSTSIZE}
685 # cleanup the system at last
688 complete $(basename $0) $SECONDS