3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
29 check_and_setup_lustre
31 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
32 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
34 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
35 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
38 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
39 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
43 MDT_DEV="${FSNAME}-MDT0000"
44 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
45 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
46 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
47 SHOW_SCRUB="do_facet $SINGLEMDS \
48 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
49 MOUNT_OPTS_SCRUB="-o user_xattr"
50 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
60 echo "preparing... ${nfiles} files will be created."
62 cp $LUSTRE/tests/*.sh $DIR/$tdir/
63 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
64 error "createmany failed"; }
67 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
68 echo "stop $SINGLEMDS"
69 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
74 echo "start $SINGLEMDS without disabling OI scrub"
75 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
76 error "(1) Fail to start MDS!"
78 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
79 [ "$STATUS" == "init" ] ||
80 error "(2) Expect 'init', but got '$STATUS'"
82 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
83 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
85 mount_client $MOUNT || error "(4) Fail to start client!"
87 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
88 error "(5) File diff failed unexpected!"
90 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
94 echo "start $SINGLEMDS without disabling OI scrub"
95 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
96 error "(1) Fail to start MDS!"
98 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
99 [ "$STATUS" == "init" ] ||
100 error "(2) Expect 'init', but got '$STATUS'"
102 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
103 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
105 mount_client $MOUNT || error "(4) Fail to start client!"
107 #define OBD_FAIL_OSD_FID_MAPPING 0x193
108 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
109 # update .lustre OI mapping
111 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
113 umount_client $MOUNT || error "(5) Fail to stop client!"
115 echo "stop $SINGLEMDS"
116 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
118 echo "start $SINGLEMDS with disabling OI scrub"
119 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
120 error "(7) Fail to start MDS!"
122 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
123 [ "$STATUS" == "init" ] ||
124 error "(8) Expect 'init', but got '$STATUS'"
126 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
127 [ "$FLAGS" == "inconsistent" ] ||
128 error "(9) Expect 'inconsistent', but got '$FLAGS'"
130 run_test 1a "Auto trigger initial OI scrub when server mounts"
134 mds_remove_ois || error "(1) Fail to remove/recreate!"
136 echo "start $SINGLEMDS without disabling OI scrub"
137 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
138 error "(2) Fail to start MDS!"
141 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
142 [ "$STATUS" == "completed" ] ||
143 error "(3) Expect 'completed', but got '$STATUS'"
145 mount_client $MOUNT || error "(4) Fail to start client!"
147 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
148 error "(5) File diff failed unexpected!"
150 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
155 # OI files to be removed:
158 # idx 2: oi.16.{2,4,8,16,32}
159 # idx 3: oi.16.{3,9,27}
160 # idx 5: oi.16.{5,25}
161 # idx 7: oi.16.{7,49}
162 for index in 0 1 2 3 5 7; do
164 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
166 echo "start $SINGLEMDS with disabling OI scrub"
167 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
168 /dev/null || error "(2) Fail to start MDS!"
170 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
171 [ "$FLAGS" == "recreated" ] ||
172 error "(3) Expect 'recreated', but got '$FLAGS'"
174 $START_SCRUB || error "(4) Fail to start OI scrub!"
176 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
177 [ "$STATUS" == "completed" ] ||
178 error "(5) Expect 'completed', but got '$STATUS'"
180 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
182 error "(6) Expect empty flags, but got '$FLAGS'"
185 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
189 mds_backup_restore || error "(1) Fail to backup/restore!"
191 echo "start $SINGLEMDS without disabling OI scrub"
192 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
193 error "(2) Fail to start MDS!"
196 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
197 [ "$STATUS" == "completed" ] ||
198 error "(3) Expect 'completed', but got '$STATUS'"
200 mount_client $MOUNT || error "(4) Fail to start client!"
202 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
203 error "(5) File diff failed unexpected!"
205 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
209 mds_backup_restore || error "(1) Fail to backup/restore!"
211 echo "start $SINGLEMDS with disabling OI scrub"
212 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
213 error "(2) Fail to start MDS!"
215 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
216 [ "$STATUS" == "init" ] ||
217 error "(3) Expect 'init', but got '$STATUS'"
219 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
220 [ "$FLAGS" == "inconsistent" ] ||
221 error "(4) Expect 'inconsistent', but got '$FLAGS'"
225 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
229 mds_backup_restore || error "(1) Fail to backup/restore!"
231 echo "start $SINGLEMDS with disabling OI scrub"
232 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
233 error "(2) Fail to start MDS!"
235 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
236 [ "$STATUS" == "init" ] ||
237 error "(3) Expect 'init', but got '$STATUS'"
239 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
240 [ "$FLAGS" == "inconsistent" ] ||
241 error "(4) Expect 'inconsistent', but got '$FLAGS'"
243 mount_client $MOUNT || error "(5) Fail to start client!"
245 do_facet $SINGLEMDS \
246 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
247 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
248 error "(6) File diff failed unexpected!"
251 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
252 [ "$STATUS" == "completed" ] ||
253 error "(7) Expect 'completed', but got '$STATUS'"
255 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
259 mds_backup_restore || error "(1) Fail to backup/restore!"
261 echo "start $SINGLEMDS with disabling OI scrub"
262 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
263 error "(2) Fail to start MDS!"
265 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
266 [ "$STATUS" == "init" ] ||
267 error "(3) Expect 'init', but got '$STATUS'"
269 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
270 [ "$FLAGS" == "inconsistent" ] ||
271 error "(4) Expect 'inconsistent', but got '$FLAGS'"
273 mount_client $MOUNT || error "(5) Fail to start client!"
275 do_facet $SINGLEMDS \
276 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
277 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
278 do_facet $SINGLEMDS $LCTL set_param fail_val=3
279 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
280 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
281 error "(6) File diff failed unexpected!"
283 umount_client $MOUNT || error "(7) Fail to stop client!"
285 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
286 [ "$STATUS" == "scanning" ] ||
287 error "(8) Expect 'scanning', but got '$STATUS'"
289 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
290 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
292 echo "stop $SINGLEMDS"
293 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
295 echo "start $SINGLEMDS with disabling OI scrub"
296 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
297 error "(10) Fail to start MDS!"
299 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
300 [ "$STATUS" == "crashed" ] ||
301 error "(11) Expect 'crashed', but got '$STATUS'"
303 echo "stop $SINGLEMDS"
304 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
306 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
307 do_facet $SINGLEMDS $LCTL set_param fail_val=3
308 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
309 echo "start $SINGLEMDS without disabling OI scrub"
310 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
311 error "(13) Fail to start MDS!"
313 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
314 [ "$STATUS" == "scanning" ] ||
315 error "(14) Expect 'scanning', but got '$STATUS'"
317 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
318 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
320 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
321 [ "$STATUS" == "failed" ] ||
322 error "(15) Expect 'failed', but got '$STATUS'"
324 mount_client $MOUNT || error "(16) Fail to start client!"
326 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
327 do_facet $SINGLEMDS $LCTL set_param fail_val=3
328 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
329 stat $DIR/$tdir/${tfile}1000 ||
330 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
332 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
333 [ "$STATUS" == "scanning" ] ||
334 error "(18) Expect 'scanning', but got '$STATUS'"
336 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
337 do_facet $SINGLEMDS $LCTL set_param fail_val=0
339 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
340 [ "$STATUS" == "completed" ] ||
341 error "(19) Expect 'completed', but got '$STATUS'"
343 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
344 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
346 run_test 5 "OI scrub state machine"
350 mds_backup_restore || error "(1) Fail to backup/restore!"
352 echo "start $SINGLEMDS with disabling OI scrub"
353 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
354 error "(2) Fail to start MDS!"
356 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
357 [ "$STATUS" == "init" ] ||
358 error "(3) Expect 'init', but got '$STATUS'"
360 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
361 [ "$FLAGS" == "inconsistent" ] ||
362 error "(4) Expect 'inconsistent', but got '$FLAGS'"
364 mount_client $MOUNT || error "(5) Fail to start client!"
366 do_facet $SINGLEMDS \
367 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
368 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
369 do_facet $SINGLEMDS $LCTL set_param fail_val=3
370 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
371 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
372 error "(6) File diff failed unexpected!"
374 # Sleep 5 sec to guarantee at least one object processed by OI scrub
376 # Fail the OI scrub to guarantee there is at least one checkpoint
377 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
378 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
380 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
381 [ "$STATUS" == "failed" ] ||
382 error "(7) Expect 'failed', but got '$STATUS'"
384 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
385 do_facet $SINGLEMDS $LCTL set_param fail_val=3
386 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
387 # stat will re-trigger OI scrub
388 stat $DIR/$tdir/${tfile}800 ||
389 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
391 umount_client $MOUNT || error "(9) Fail to stop client!"
393 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
394 [ "$STATUS" == "scanning" ] ||
395 error "(10) Expect 'scanning', but got '$STATUS'"
397 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
398 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
400 local POSITION0=$($SHOW_SCRUB |
401 awk '/^last_checkpoint_position/ {print $2}')
402 POSITION0=$((POSITION0 + 1))
404 echo "stop $SINGLEMDS"
405 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
407 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
408 do_facet $SINGLEMDS $LCTL set_param fail_val=3
409 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
410 echo "start $SINGLEMDS without disabling OI scrub"
411 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
412 error "(12) Fail to start MDS!"
414 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
415 [ "$STATUS" == "scanning" ] ||
416 error "(13) Expect 'scanning', but got '$STATUS'"
418 local POSITION1=$($SHOW_SCRUB |
419 awk '/^latest_start_position/ {print $2}')
420 [ $POSITION0 -eq $POSITION1 ] ||
421 error "(14) Expect position: $POSITION0, but got $POSITION1"
423 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
424 do_facet $SINGLEMDS $LCTL set_param fail_val=0
426 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
427 [ "$STATUS" == "completed" ] ||
428 error "(15) Expect 'completed', but got '$STATUS'"
430 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
431 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
433 run_test 6 "OI scrub resumes from last checkpoint"
437 mds_backup_restore || error "(1) Fail to backup/restore!"
439 echo "start $SINGLEMDS with disabling OI scrub"
440 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
441 error "(2) Fail to start MDS!"
443 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
444 [ "$STATUS" == "init" ] ||
445 error "(3) Expect 'init', but got '$STATUS'"
447 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
448 [ "$FLAGS" == "inconsistent" ] ||
449 error "(4) Expect 'inconsistent', but got '$FLAGS'"
451 mount_client $MOUNT || error "(5) Fail to start client!"
453 do_facet $SINGLEMDS \
454 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
455 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
456 do_facet $SINGLEMDS $LCTL set_param fail_val=3
457 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
458 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
459 error "(6) File diff failed unexpected!"
461 stat $DIR/$tdir/${tfile}300 ||
462 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
464 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
465 [ "$STATUS" == "scanning" ] ||
466 error "(8) Expect 'scanning', but got '$STATUS'"
468 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
469 [ "$FLAGS" == "inconsistent,auto" ] ||
470 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
472 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
473 do_facet $SINGLEMDS $LCTL set_param fail_val=0
475 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
476 [ "$STATUS" == "completed" ] ||
477 error "(10) Expect 'completed', but got '$STATUS'"
479 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
480 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
482 run_test 7 "System is available during OI scrub scanning"
486 mds_backup_restore || error "(1) Fail to backup/restore!"
488 echo "start $SINGLEMDS with disabling OI scrub"
489 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
490 error "(2) Fail to start MDS!"
492 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
493 [ "$STATUS" == "init" ] ||
494 error "(3) Expect 'init', but got '$STATUS'"
496 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
497 [ "$FLAGS" == "inconsistent" ] ||
498 error "(4) Expect 'inconsistent', but got '$FLAGS'"
500 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
501 do_facet $SINGLEMDS $LCTL set_param fail_val=3
502 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
503 $START_SCRUB || error "(5) Fail to start OI scrub!"
505 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
506 [ "$STATUS" == "scanning" ] ||
507 error "(6) Expect 'scanning', but got '$STATUS'"
509 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
511 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
512 [ "$STATUS" == "stopped" ] ||
513 error "(8) Expect 'stopped', but got '$STATUS'"
515 $START_SCRUB || error "(9) Fail to start OI scrub!"
517 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
518 [ "$STATUS" == "scanning" ] ||
519 error "(10) Expect 'scanning', but got '$STATUS'"
521 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
522 do_facet $SINGLEMDS $LCTL set_param fail_val=0
524 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
525 [ "$STATUS" == "completed" ] ||
526 error "(11) Expect 'completed', but got '$STATUS'"
528 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
529 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
531 run_test 8 "Control OI scrub manually"
534 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
535 skip "Testing on UP system, the speed may be inaccurate."
540 mds_backup_restore || error "(1) Fail to backup/restore!"
542 echo "start $SINGLEMDS with disabling OI scrub"
543 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
544 error "(2) Fail to start MDS!"
546 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
547 [ "$STATUS" == "init" ] ||
548 error "(3) Expect 'init', but got '$STATUS'"
550 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
551 [ "$FLAGS" == "inconsistent" ] ||
552 error "(4) Expect 'inconsistent', but got '$FLAGS'"
554 local BASE_SPEED1=100
556 # OI scrub should run with full speed under inconsistent case
557 $START_SCRUB -s $BASE_SPEED1 || error "(5) Fail to start OI scrub!"
560 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
561 [ "$STATUS" == "completed" ] ||
562 error "(6) Expect 'completed', but got '$STATUS'"
564 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
565 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
567 # OI scrub should run with limited speed under non-inconsistent case
568 $START_SCRUB -s $BASE_SPEED1 -r || error "(8) Fail to start OI scrub!"
571 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
572 [ "$STATUS" == "scanning" ] ||
573 error "(9) Expect 'scanning', but got '$STATUS'"
575 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
577 # Do NOT ignore that there are 1024 pre-fetched items. And there
578 # may be time error, normally it should be less than 2 seconds.
579 # We allow another 20% schedule error.
580 local PRE_FETCHED=1024
582 # MAX_MARGIN = 1.2 = 12 / 10
583 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
584 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
585 [ $SPEED -lt $MAX_SPEED ] ||
586 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
589 local BASE_SPEED2=300
591 do_facet $SINGLEMDS \
592 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
595 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
596 # MIN_MARGIN = 0.8 = 8 / 10
597 local MIN_SPEED=$(((PRE_FETCHED + \
598 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
599 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
600 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
601 [ $SPEED -gt $MIN_SPEED ] ||
602 error "(11) Got speed $SPEED, expected more than $MIN_SPEED"
604 # MAX_MARGIN = 1.2 = 12 / 10
605 MAX_SPEED=$(((PRE_FETCHED + \
606 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
607 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
608 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
609 [ $SPEED -lt $MAX_SPEED ] ||
610 error "(12) Got speed $SPEED, expected less than $MAX_SPEED"
612 do_facet $SINGLEMDS \
613 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
615 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
616 [ "$STATUS" == "completed" ] ||
617 error "(13) Expect 'completed', but got '$STATUS'"
619 run_test 9 "OI scrub speed control"
623 mds_backup_restore || error "(1) Fail to backup/restore!"
625 echo "start $SINGLEMDS with disabling OI scrub"
626 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
627 error "(2) Fail to start MDS!"
629 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
630 [ "$STATUS" == "init" ] ||
631 error "(3) Expect 'init', but got '$STATUS'"
633 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
634 [ "$FLAGS" == "inconsistent" ] ||
635 error "(4) Expect 'inconsistent', but got '$FLAGS'"
637 mount_client $MOUNT || error "(5) Fail to start client!"
639 do_facet $SINGLEMDS \
640 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
641 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
642 do_facet $SINGLEMDS $LCTL set_param fail_val=3
643 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
644 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
645 error "(6) File diff failed unexpected!"
647 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
648 [ "$STATUS" == "scanning" ] ||
649 error "(7) Expect 'scanning', but got '$STATUS'"
651 umount_client $MOUNT || error "(8) Fail to stop client!"
653 echo "stop $SINGLEMDS"
654 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
656 echo "start $SINGLEMDS with disabling OI scrub"
657 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
658 error "(10) Fail to start MDS!"
660 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
661 [ "$STATUS" == "paused" ] ||
662 error "(11) Expect 'paused', but got '$STATUS'"
664 echo "stop $SINGLEMDS"
665 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
667 echo "start $SINGLEMDS without disabling OI scrub"
668 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
669 error "(13) Fail to start MDS!"
671 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
672 [ "$STATUS" == "scanning" ] ||
673 error "(14) Expect 'scanning', but got '$STATUS'"
675 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
676 do_facet $SINGLEMDS $LCTL set_param fail_val=0
678 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
679 [ "$STATUS" == "completed" ] ||
680 error "(15) Expect 'completed', but got '$STATUS'"
682 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
683 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
685 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
689 mds_backup_restore || error "(1) Fail to backup/restore!"
691 echo "start $SINGLEMDS with disabling OI scrub"
692 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
693 error "(2) Fail to start MDS!"
695 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
696 [ "$STATUS" == "init" ] ||
697 error "(3) Expect 'init', but got '$STATUS'"
699 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
700 [ "$FLAGS" == "inconsistent" ] ||
701 error "(4) Expect 'inconsistent', but got '$FLAGS'"
703 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
704 do_facet $SINGLEMDS $LCTL set_param fail_val=3
705 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
707 $START_SCRUB || error "(5) Fail to start OI scrub!"
709 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
710 [ "$STATUS" == "scanning" ] ||
711 error "(6) Expect 'scanning', but got '$STATUS'"
713 echo "stop $SINGLEMDS"
714 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
716 echo "start $SINGLEMDS with disabling OI scrub"
717 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
718 error "(8) Fail to start MDS!"
720 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
721 [ "$STATUS" == "paused" ] ||
722 error "(9) Expect 'paused', but got '$STATUS'"
724 echo "stop $SINGLEMDS"
725 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
727 echo "start $SINGLEMDS without disabling OI scrub"
728 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
729 error "(11) Fail to start MDS!"
731 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
732 [ "$STATUS" == "scanning" ] ||
733 error "(12) Expect 'scanning', but got '$STATUS'"
735 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
736 do_facet $SINGLEMDS $LCTL set_param fail_val=0
738 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
739 [ "$STATUS" == "completed" ] ||
740 error "(13) Expect 'completed', but got '$STATUS'"
742 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
743 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
745 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
751 formatall > /dev/null
756 local tname=`date +%s`
757 rm -rf $MOUNT/$tname > /dev/null
758 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
760 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
762 # reset OI scrub start point by force
763 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
765 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
766 [ "$STATUS" == "completed" ] ||
767 error "(4) Expect 'completed', but got '$STATUS'"
769 # OI scrub should skip the new created objects for the first accessing
770 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
771 # notice we're creating a new llog for every OST on every startup
772 # new features can make this even less stable, so we only check
773 # that the number of skipped files is less than 2x the number of files
774 local MAXIMUM=$((CREATED * 2))
775 local MINIMUM=$((CREATED + 1)) # files + directory
776 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
777 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
779 # reset OI scrub start point by force
780 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
782 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
783 [ "$STATUS" == "completed" ] ||
784 error "(7) Expect 'completed', but got '$STATUS'"
786 # OI scrub should skip the new created object only once
787 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
788 [ $SKIPPED -eq 0 ] ||
789 error "(8) Expect 0 objects skipped, but got $SKIPPED"
791 rm -rf $MOUNT/$tname > /dev/null
793 run_test 11 "OI scrub skips the new created objects only once"
795 # restore MDS/OST size
796 MDSSIZE=${SAVED_MDSSIZE}
797 OSTSIZE=${SAVED_OSTSIZE}
799 # cleanup the system at last