3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
21 skip "test OI scrub only for ldiskfs" && exit 0
22 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
23 skip "Need MDS version at least 2.2.90" && exit 0
24 require_dsh_mds || exit 0
26 SAVED_MDSSIZE=${MDSSIZE}
27 SAVED_OSTSIZE=${OSTSIZE}
28 # use small MDS + OST size to speed formatting time
29 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
33 check_and_setup_lustre
36 MDT_DEV="${FSNAME}-MDT0000"
37 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
38 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
39 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
40 SHOW_SCRUB="do_facet $SINGLEMDS \
41 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
42 MOUNT_OPTS_SCRUB="-o user_xattr"
43 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
53 echo "preparing... ${nfiles} files will be created."
55 cp $LUSTRE/tests/*.sh $DIR/$tdir/
56 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
57 error "createmany failed"; }
60 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
61 echo "stop $SINGLEMDS"
62 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
67 echo "start $SINGLEMDS without disabling OI scrub"
68 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
69 error "(1) Fail to start MDS!"
71 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
72 [ "$STATUS" == "init" ] ||
73 error "(2) Expect 'init', but got '$STATUS'"
75 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
76 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
78 mount_client $MOUNT || error "(4) Fail to start client!"
80 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
81 error "(5) File diff failed unexpected!"
83 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
87 echo "start $SINGLEMDS without disabling OI scrub"
88 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
89 error "(1) Fail to start MDS!"
91 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
92 [ "$STATUS" == "init" ] ||
93 error "(2) Expect 'init', but got '$STATUS'"
95 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
96 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
98 mount_client $MOUNT || error "(4) Fail to start client!"
100 #define OBD_FAIL_OSD_FID_MAPPING 0x193
101 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
102 # update .lustre OI mapping
104 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
106 umount_client $MOUNT || error "(5) Fail to stop client!"
108 echo "stop $SINGLEMDS"
109 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
111 echo "start $SINGLEMDS with disabling OI scrub"
112 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
113 error "(7) Fail to start MDS!"
115 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
116 [ "$STATUS" == "init" ] ||
117 error "(8) Expect 'init', but got '$STATUS'"
119 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
120 [ "$FLAGS" == "inconsistent" ] ||
121 error "(9) Expect 'inconsistent', but got '$FLAGS'"
123 run_test 1a "Auto trigger initial OI scrub when server mounts"
127 mds_remove_ois || error "(1) Fail to remove/recreate!"
129 echo "start $SINGLEMDS without disabling OI scrub"
130 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
131 error "(2) Fail to start MDS!"
134 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
135 [ "$STATUS" == "completed" ] ||
136 error "(3) Expect 'completed', but got '$STATUS'"
138 mount_client $MOUNT || error "(4) Fail to start client!"
140 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
141 error "(5) File diff failed unexpected!"
143 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
148 # OI files to be removed:
151 # idx 2: oi.16.{2,4,8,16,32}
152 # idx 3: oi.16.{3,9,27}
153 # idx 5: oi.16.{5,25}
154 # idx 7: oi.16.{7,49}
155 for index in 0 1 2 3 5 7; do
157 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
159 echo "start $SINGLEMDS with disabling OI scrub"
160 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
161 /dev/null || error "(2) Fail to start MDS!"
163 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
164 [ "$FLAGS" == "recreated" ] ||
165 error "(3) Expect 'recreated', but got '$FLAGS'"
167 $START_SCRUB || error "(4) Fail to start OI scrub!"
169 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
170 [ "$STATUS" == "completed" ] ||
171 error "(5) Expect 'completed', but got '$STATUS'"
173 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
175 error "(6) Expect empty flags, but got '$FLAGS'"
178 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
182 mds_backup_restore || error "(1) Fail to backup/restore!"
184 echo "start $SINGLEMDS without disabling OI scrub"
185 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
186 error "(2) Fail to start MDS!"
189 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
190 [ "$STATUS" == "completed" ] ||
191 error "(3) Expect 'completed', but got '$STATUS'"
193 mount_client $MOUNT || error "(4) Fail to start client!"
195 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
196 error "(5) File diff failed unexpected!"
198 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
202 mds_backup_restore || error "(1) Fail to backup/restore!"
204 echo "start $SINGLEMDS with disabling OI scrub"
205 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
206 error "(2) Fail to start MDS!"
208 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
209 [ "$STATUS" == "init" ] ||
210 error "(3) Expect 'init', but got '$STATUS'"
212 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
213 [ "$FLAGS" == "inconsistent" ] ||
214 error "(4) Expect 'inconsistent', but got '$FLAGS'"
218 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
222 mds_backup_restore || error "(1) Fail to backup/restore!"
224 echo "start $SINGLEMDS with disabling OI scrub"
225 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
226 error "(2) Fail to start MDS!"
228 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
229 [ "$STATUS" == "init" ] ||
230 error "(3) Expect 'init', but got '$STATUS'"
232 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
233 [ "$FLAGS" == "inconsistent" ] ||
234 error "(4) Expect 'inconsistent', but got '$FLAGS'"
236 mount_client $MOUNT || error "(5) Fail to start client!"
238 do_facet $SINGLEMDS \
239 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
240 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
241 error "(6) File diff failed unexpected!"
244 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
245 [ "$STATUS" == "completed" ] ||
246 error "(7) Expect 'completed', but got '$STATUS'"
248 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
252 mds_backup_restore || error "(1) Fail to backup/restore!"
254 echo "start $SINGLEMDS with disabling OI scrub"
255 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
256 error "(2) Fail to start MDS!"
258 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
259 [ "$STATUS" == "init" ] ||
260 error "(3) Expect 'init', but got '$STATUS'"
262 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
263 [ "$FLAGS" == "inconsistent" ] ||
264 error "(4) Expect 'inconsistent', but got '$FLAGS'"
266 mount_client $MOUNT || error "(5) Fail to start client!"
268 do_facet $SINGLEMDS \
269 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
270 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
271 do_facet $SINGLEMDS $LCTL set_param fail_val=3
272 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
273 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
274 error "(6) File diff failed unexpected!"
276 umount_client $MOUNT || error "(7) Fail to stop client!"
278 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
279 [ "$STATUS" == "scanning" ] ||
280 error "(8) Expect 'scanning', but got '$STATUS'"
282 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
283 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
285 echo "stop $SINGLEMDS"
286 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
288 echo "start $SINGLEMDS with disabling OI scrub"
289 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
290 error "(10) Fail to start MDS!"
292 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
293 [ "$STATUS" == "crashed" ] ||
294 error "(11) Expect 'crashed', but got '$STATUS'"
296 echo "stop $SINGLEMDS"
297 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
299 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
300 do_facet $SINGLEMDS $LCTL set_param fail_val=3
301 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
302 echo "start $SINGLEMDS without disabling OI scrub"
303 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
304 error "(13) Fail to start MDS!"
306 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
307 [ "$STATUS" == "scanning" ] ||
308 error "(14) Expect 'scanning', but got '$STATUS'"
310 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
311 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
313 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
314 [ "$STATUS" == "failed" ] ||
315 error "(15) Expect 'failed', but got '$STATUS'"
317 mount_client $MOUNT || error "(16) Fail to start client!"
319 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
320 do_facet $SINGLEMDS $LCTL set_param fail_val=3
321 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
322 stat $DIR/$tdir/${tfile}1000 ||
323 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
325 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
326 [ "$STATUS" == "scanning" ] ||
327 error "(18) Expect 'scanning', but got '$STATUS'"
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
330 do_facet $SINGLEMDS $LCTL set_param fail_val=0
332 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
333 [ "$STATUS" == "completed" ] ||
334 error "(19) Expect 'completed', but got '$STATUS'"
336 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
337 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
339 run_test 5 "OI scrub state machine"
343 mds_backup_restore || error "(1) Fail to backup/restore!"
345 echo "start $SINGLEMDS with disabling OI scrub"
346 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
347 error "(2) Fail to start MDS!"
349 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
350 [ "$STATUS" == "init" ] ||
351 error "(3) Expect 'init', but got '$STATUS'"
353 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
354 [ "$FLAGS" == "inconsistent" ] ||
355 error "(4) Expect 'inconsistent', but got '$FLAGS'"
357 mount_client $MOUNT || error "(5) Fail to start client!"
359 do_facet $SINGLEMDS \
360 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
361 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
362 do_facet $SINGLEMDS $LCTL set_param fail_val=3
363 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
364 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
365 error "(6) File diff failed unexpected!"
367 # Sleep 5 sec to guarantee at least one object processed by OI scrub
369 # Fail the OI scrub to guarantee there is at least one checkpoint
370 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
371 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
373 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
374 [ "$STATUS" == "failed" ] ||
375 error "(7) Expect 'failed', but got '$STATUS'"
377 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
378 do_facet $SINGLEMDS $LCTL set_param fail_val=3
379 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
380 # stat will re-trigger OI scrub
381 stat $DIR/$tdir/${tfile}800 ||
382 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
384 umount_client $MOUNT || error "(9) Fail to stop client!"
386 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
387 [ "$STATUS" == "scanning" ] ||
388 error "(10) Expect 'scanning', but got '$STATUS'"
390 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
391 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
393 local POSITION0=$($SHOW_SCRUB |
394 awk '/^last_checkpoint_position/ {print $2}')
395 POSITION0=$((POSITION0 + 1))
397 echo "stop $SINGLEMDS"
398 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
400 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
401 do_facet $SINGLEMDS $LCTL set_param fail_val=3
402 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
403 echo "start $SINGLEMDS without disabling OI scrub"
404 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
405 error "(12) Fail to start MDS!"
407 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
408 [ "$STATUS" == "scanning" ] ||
409 error "(13) Expect 'scanning', but got '$STATUS'"
411 local POSITION1=$($SHOW_SCRUB |
412 awk '/^latest_start_position/ {print $2}')
413 [ $POSITION0 -eq $POSITION1 ] ||
414 error "(14) Expect position: $POSITION0, but got $POSITION1"
416 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
417 do_facet $SINGLEMDS $LCTL set_param fail_val=0
419 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
420 [ "$STATUS" == "completed" ] ||
421 error "(15) Expect 'completed', but got '$STATUS'"
423 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
424 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
426 run_test 6 "OI scrub resumes from last checkpoint"
430 mds_backup_restore || error "(1) Fail to backup/restore!"
432 echo "start $SINGLEMDS with disabling OI scrub"
433 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
434 error "(2) Fail to start MDS!"
436 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
437 [ "$STATUS" == "init" ] ||
438 error "(3) Expect 'init', but got '$STATUS'"
440 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
441 [ "$FLAGS" == "inconsistent" ] ||
442 error "(4) Expect 'inconsistent', but got '$FLAGS'"
444 mount_client $MOUNT || error "(5) Fail to start client!"
446 do_facet $SINGLEMDS \
447 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
448 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
449 do_facet $SINGLEMDS $LCTL set_param fail_val=3
450 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
451 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
452 error "(6) File diff failed unexpected!"
454 stat $DIR/$tdir/${tfile}300 ||
455 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
457 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
458 [ "$STATUS" == "scanning" ] ||
459 error "(8) Expect 'scanning', but got '$STATUS'"
461 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
462 [ "$FLAGS" == "inconsistent,auto" ] ||
463 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
465 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
466 do_facet $SINGLEMDS $LCTL set_param fail_val=0
468 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "completed" ] ||
470 error "(10) Expect 'completed', but got '$STATUS'"
472 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
473 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
475 run_test 7 "System is available during OI scrub scanning"
479 mds_backup_restore || error "(1) Fail to backup/restore!"
481 echo "start $SINGLEMDS with disabling OI scrub"
482 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
483 error "(2) Fail to start MDS!"
485 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
486 [ "$STATUS" == "init" ] ||
487 error "(3) Expect 'init', but got '$STATUS'"
489 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
490 [ "$FLAGS" == "inconsistent" ] ||
491 error "(4) Expect 'inconsistent', but got '$FLAGS'"
493 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
494 do_facet $SINGLEMDS $LCTL set_param fail_val=3
495 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
496 $START_SCRUB || error "(5) Fail to start OI scrub!"
498 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
499 [ "$STATUS" == "scanning" ] ||
500 error "(6) Expect 'scanning', but got '$STATUS'"
502 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
504 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
505 [ "$STATUS" == "stopped" ] ||
506 error "(8) Expect 'stopped', but got '$STATUS'"
508 $START_SCRUB || error "(9) Fail to start OI scrub!"
510 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
511 [ "$STATUS" == "scanning" ] ||
512 error "(10) Expect 'scanning', but got '$STATUS'"
514 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
515 do_facet $SINGLEMDS $LCTL set_param fail_val=0
517 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
518 [ "$STATUS" == "completed" ] ||
519 error "(11) Expect 'completed', but got '$STATUS'"
521 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
522 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
524 run_test 8 "Control OI scrub manually"
527 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
528 skip "Testing on UP system, the speed may be inaccurate."
533 mds_backup_restore || error "(1) Fail to backup/restore!"
535 echo "start $SINGLEMDS with disabling OI scrub"
536 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
537 error "(2) Fail to start MDS!"
539 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
540 [ "$STATUS" == "init" ] ||
541 error "(3) Expect 'init', but got '$STATUS'"
543 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
544 [ "$FLAGS" == "inconsistent" ] ||
545 error "(4) Expect 'inconsistent', but got '$FLAGS'"
547 # OI scrub should run with full speed under inconsistent case
548 $START_SCRUB -s 100 || error "(5) Fail to start OI scrub!"
551 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
552 [ "$STATUS" == "completed" ] ||
553 error "(6) Expect 'completed', but got '$STATUS'"
555 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
556 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
558 # OI scrub should run with limited speed under non-inconsistent case
559 $START_SCRUB -s 100 -r || error "(8) Fail to start OI scrub!"
562 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
563 [ "$STATUS" == "scanning" ] ||
564 error "(9) Expect 'scanning', but got '$STATUS'"
566 # Do NOT ignore that there are 1024 pre-fetched items.
567 # So the max speed may be (1024 + 100 * 10) / 10.
568 # And there may be time error, so the max speed may be more large.
569 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
570 [ $SPEED -gt 220 ] &&
571 error "(10) Unexpected speed $SPEED, should not more than 220"
574 do_facet $SINGLEMDS \
575 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 300
578 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
579 [ $SPEED -lt 220 ] &&
580 error "(11) Unexpected speed $SPEED, should not less than 220"
582 [ $SPEED -gt 300 ] &&
583 error "(12) Unexpected speed $SPEED, should not more than 300"
585 do_facet $SINGLEMDS \
586 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
588 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
589 [ "$STATUS" == "completed" ] ||
590 error "(13) Expect 'completed', but got '$STATUS'"
592 run_test 9 "OI scrub speed control"
596 mds_backup_restore || error "(1) Fail to backup/restore!"
598 echo "start $SINGLEMDS with disabling OI scrub"
599 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
600 error "(2) Fail to start MDS!"
602 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
603 [ "$STATUS" == "init" ] ||
604 error "(3) Expect 'init', but got '$STATUS'"
606 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
607 [ "$FLAGS" == "inconsistent" ] ||
608 error "(4) Expect 'inconsistent', but got '$FLAGS'"
610 mount_client $MOUNT || error "(5) Fail to start client!"
612 do_facet $SINGLEMDS \
613 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
614 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
615 do_facet $SINGLEMDS $LCTL set_param fail_val=3
616 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
617 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
618 error "(6) File diff failed unexpected!"
620 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
621 [ "$STATUS" == "scanning" ] ||
622 error "(7) Expect 'scanning', but got '$STATUS'"
624 umount_client $MOUNT || error "(8) Fail to stop client!"
626 echo "stop $SINGLEMDS"
627 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
629 echo "start $SINGLEMDS with disabling OI scrub"
630 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
631 error "(10) Fail to start MDS!"
633 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
634 [ "$STATUS" == "paused" ] ||
635 error "(11) Expect 'paused', but got '$STATUS'"
637 echo "stop $SINGLEMDS"
638 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
640 echo "start $SINGLEMDS without disabling OI scrub"
641 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
642 error "(13) Fail to start MDS!"
644 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
645 [ "$STATUS" == "scanning" ] ||
646 error "(14) Expect 'scanning', but got '$STATUS'"
648 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
649 do_facet $SINGLEMDS $LCTL set_param fail_val=0
651 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
652 [ "$STATUS" == "completed" ] ||
653 error "(15) Expect 'completed', but got '$STATUS'"
655 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
656 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
658 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
662 mds_backup_restore || error "(1) Fail to backup/restore!"
664 echo "start $SINGLEMDS with disabling OI scrub"
665 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
666 error "(2) Fail to start MDS!"
668 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
669 [ "$STATUS" == "init" ] ||
670 error "(3) Expect 'init', but got '$STATUS'"
672 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
673 [ "$FLAGS" == "inconsistent" ] ||
674 error "(4) Expect 'inconsistent', but got '$FLAGS'"
676 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
677 do_facet $SINGLEMDS $LCTL set_param fail_val=3
678 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
680 $START_SCRUB || error "(5) Fail to start OI scrub!"
682 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
683 [ "$STATUS" == "scanning" ] ||
684 error "(6) Expect 'scanning', but got '$STATUS'"
686 echo "stop $SINGLEMDS"
687 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
689 echo "start $SINGLEMDS with disabling OI scrub"
690 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
691 error "(8) Fail to start MDS!"
693 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
694 [ "$STATUS" == "paused" ] ||
695 error "(9) Expect 'paused', but got '$STATUS'"
697 echo "stop $SINGLEMDS"
698 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
700 echo "start $SINGLEMDS without disabling OI scrub"
701 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
702 error "(11) Fail to start MDS!"
704 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
705 [ "$STATUS" == "scanning" ] ||
706 error "(12) Expect 'scanning', but got '$STATUS'"
708 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
709 do_facet $SINGLEMDS $LCTL set_param fail_val=0
711 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
712 [ "$STATUS" == "completed" ] ||
713 error "(13) Expect 'completed', but got '$STATUS'"
715 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
716 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
718 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
727 local tname=`date +%s`
728 rm -rf $MOUNT/$tname > /dev/null
729 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
731 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
733 # reset OI scrub start point by force
734 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
736 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
737 [ "$STATUS" == "completed" ] ||
738 error "(4) Expect 'completed', but got '$STATUS'"
740 # OI scrub should skip the new created objects for the first accessing
741 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
742 # notice we're creating a new llog for every OST on every startup
743 # new features can make this even less stable, so we only check
744 # that the number of skipped files is less than 1.5x the number of files
745 local MAXIMUM=$((CREATED * 3 / 2))
746 local MINIMUM=$((CREATED + 1)) # files + directory
747 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
748 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
750 # reset OI scrub start point by force
751 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
753 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
754 [ "$STATUS" == "completed" ] ||
755 error "(7) Expect 'completed', but got '$STATUS'"
757 # OI scrub should skip the new created object only once
758 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
759 [ $SKIPPED -eq 0 ] ||
760 error "(8) Expect 0 objects skipped, but got $SKIPPED"
762 rm -rf $MOUNT/$tname > /dev/null
764 run_test 11 "OI scrub skips the new created objects only once"
766 # restore MDS/OST size
767 MDSSIZE=${SAVED_MDSSIZE}
768 OSTSIZE=${SAVED_OSTSIZE}
770 # cleanup the system at last