3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
33 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
35 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
36 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
39 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
44 MDT_DEV="${FSNAME}-MDT0000"
45 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
46 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
47 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
48 SHOW_SCRUB="do_facet $SINGLEMDS \
49 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
50 MOUNT_OPTS_SCRUB="-o user_xattr"
51 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
61 echo "preparing... ${nfiles} files will be created."
63 cp $LUSTRE/tests/*.sh $DIR/$tdir/
64 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
65 error "createmany failed"; }
68 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
69 echo "stop $SINGLEMDS"
70 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
75 echo "start $SINGLEMDS without disabling OI scrub"
76 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
77 error "(1) Fail to start MDS!"
79 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
80 [ "$STATUS" == "init" ] ||
81 error "(2) Expect 'init', but got '$STATUS'"
83 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
84 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
86 mount_client $MOUNT || error "(4) Fail to start client!"
88 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
89 error "(5) File diff failed unexpected!"
91 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
95 echo "start $SINGLEMDS without disabling OI scrub"
96 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
97 error "(1) Fail to start MDS!"
99 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
100 [ "$STATUS" == "init" ] ||
101 error "(2) Expect 'init', but got '$STATUS'"
103 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
104 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
106 mount_client $MOUNT || error "(4) Fail to start client!"
108 #define OBD_FAIL_OSD_FID_MAPPING 0x193
109 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
110 # update .lustre OI mapping
112 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
114 umount_client $MOUNT || error "(5) Fail to stop client!"
116 echo "stop $SINGLEMDS"
117 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
119 echo "start $SINGLEMDS with disabling OI scrub"
120 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
121 error "(7) Fail to start MDS!"
123 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
124 [ "$STATUS" == "init" ] ||
125 error "(8) Expect 'init', but got '$STATUS'"
127 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
128 [ "$FLAGS" == "inconsistent" ] ||
129 error "(9) Expect 'inconsistent', but got '$FLAGS'"
131 run_test 1a "Auto trigger initial OI scrub when server mounts"
135 mds_remove_ois || error "(1) Fail to remove/recreate!"
137 echo "start $SINGLEMDS without disabling OI scrub"
138 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
139 error "(2) Fail to start MDS!"
142 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
143 [ "$STATUS" == "completed" ] ||
144 error "(3) Expect 'completed', but got '$STATUS'"
146 mount_client $MOUNT || error "(4) Fail to start client!"
148 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
149 error "(5) File diff failed unexpected!"
151 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
156 # OI files to be removed:
159 # idx 2: oi.16.{2,4,8,16,32}
160 # idx 3: oi.16.{3,9,27}
161 # idx 5: oi.16.{5,25}
162 # idx 7: oi.16.{7,49}
163 for index in 0 1 2 3 5 7; do
165 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
167 echo "start $SINGLEMDS with disabling OI scrub"
168 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
169 /dev/null || error "(2) Fail to start MDS!"
171 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
172 [ "$FLAGS" == "recreated" ] ||
173 error "(3) Expect 'recreated', but got '$FLAGS'"
175 $START_SCRUB || error "(4) Fail to start OI scrub!"
177 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
178 [ "$STATUS" == "completed" ] ||
179 error "(5) Expect 'completed', but got '$STATUS'"
181 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
183 error "(6) Expect empty flags, but got '$FLAGS'"
186 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
190 mds_backup_restore || error "(1) Fail to backup/restore!"
192 echo "start $SINGLEMDS without disabling OI scrub"
193 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
194 error "(2) Fail to start MDS!"
197 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
198 [ "$STATUS" == "completed" ] ||
199 error "(3) Expect 'completed', but got '$STATUS'"
201 mount_client $MOUNT || error "(4) Fail to start client!"
203 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
204 error "(5) File diff failed unexpected!"
206 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
210 mds_backup_restore || error "(1) Fail to backup/restore!"
212 echo "start $SINGLEMDS with disabling OI scrub"
213 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
214 error "(2) Fail to start MDS!"
216 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
217 [ "$STATUS" == "init" ] ||
218 error "(3) Expect 'init', but got '$STATUS'"
220 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
221 [ "$FLAGS" == "inconsistent" ] ||
222 error "(4) Expect 'inconsistent', but got '$FLAGS'"
226 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
230 mds_backup_restore || error "(1) Fail to backup/restore!"
232 echo "start $SINGLEMDS with disabling OI scrub"
233 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
234 error "(2) Fail to start MDS!"
236 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
237 [ "$STATUS" == "init" ] ||
238 error "(3) Expect 'init', but got '$STATUS'"
240 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
241 [ "$FLAGS" == "inconsistent" ] ||
242 error "(4) Expect 'inconsistent', but got '$FLAGS'"
244 mount_client $MOUNT || error "(5) Fail to start client!"
246 do_facet $SINGLEMDS \
247 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
248 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
249 error "(6) File diff failed unexpected!"
252 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
253 [ "$STATUS" == "completed" ] ||
254 error "(7) Expect 'completed', but got '$STATUS'"
256 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
260 mds_backup_restore || error "(1) Fail to backup/restore!"
262 echo "start $SINGLEMDS with disabling OI scrub"
263 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
264 error "(2) Fail to start MDS!"
266 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
267 [ "$STATUS" == "init" ] ||
268 error "(3) Expect 'init', but got '$STATUS'"
270 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
271 [ "$FLAGS" == "inconsistent" ] ||
272 error "(4) Expect 'inconsistent', but got '$FLAGS'"
274 mount_client $MOUNT || error "(5) Fail to start client!"
276 do_facet $SINGLEMDS \
277 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
278 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
279 do_facet $SINGLEMDS $LCTL set_param fail_val=3
280 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
281 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
282 error "(6) File diff failed unexpected!"
284 umount_client $MOUNT || error "(7) Fail to stop client!"
286 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
287 [ "$STATUS" == "scanning" ] ||
288 error "(8) Expect 'scanning', but got '$STATUS'"
290 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
291 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
293 echo "stop $SINGLEMDS"
294 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
296 echo "start $SINGLEMDS with disabling OI scrub"
297 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
298 error "(10) Fail to start MDS!"
300 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
301 [ "$STATUS" == "crashed" ] ||
302 error "(11) Expect 'crashed', but got '$STATUS'"
304 echo "stop $SINGLEMDS"
305 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
307 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
308 do_facet $SINGLEMDS $LCTL set_param fail_val=3
309 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
310 echo "start $SINGLEMDS without disabling OI scrub"
311 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
312 error "(13) Fail to start MDS!"
314 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
315 [ "$STATUS" == "scanning" ] ||
316 error "(14) Expect 'scanning', but got '$STATUS'"
318 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
319 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
321 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
322 [ "$STATUS" == "failed" ] ||
323 error "(15) Expect 'failed', but got '$STATUS'"
325 mount_client $MOUNT || error "(16) Fail to start client!"
327 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
328 do_facet $SINGLEMDS $LCTL set_param fail_val=3
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
330 stat $DIR/$tdir/${tfile}1000 ||
331 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
333 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
334 [ "$STATUS" == "scanning" ] ||
335 error "(18) Expect 'scanning', but got '$STATUS'"
337 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
338 do_facet $SINGLEMDS $LCTL set_param fail_val=0
340 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
341 [ "$STATUS" == "completed" ] ||
342 error "(19) Expect 'completed', but got '$STATUS'"
344 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
345 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
347 run_test 5 "OI scrub state machine"
351 mds_backup_restore || error "(1) Fail to backup/restore!"
353 echo "start $SINGLEMDS with disabling OI scrub"
354 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
355 error "(2) Fail to start MDS!"
357 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
358 [ "$STATUS" == "init" ] ||
359 error "(3) Expect 'init', but got '$STATUS'"
361 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
362 [ "$FLAGS" == "inconsistent" ] ||
363 error "(4) Expect 'inconsistent', but got '$FLAGS'"
365 mount_client $MOUNT || error "(5) Fail to start client!"
367 do_facet $SINGLEMDS \
368 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
369 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
370 do_facet $SINGLEMDS $LCTL set_param fail_val=3
371 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
372 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
373 error "(6) File diff failed unexpected!"
375 # Sleep 5 sec to guarantee at least one object processed by OI scrub
377 # Fail the OI scrub to guarantee there is at least one checkpoint
378 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
379 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
381 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
382 [ "$STATUS" == "failed" ] ||
383 error "(7) Expect 'failed', but got '$STATUS'"
385 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
386 do_facet $SINGLEMDS $LCTL set_param fail_val=3
387 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
388 # stat will re-trigger OI scrub
389 stat $DIR/$tdir/${tfile}800 ||
390 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
392 umount_client $MOUNT || error "(9) Fail to stop client!"
394 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
395 [ "$STATUS" == "scanning" ] ||
396 error "(10) Expect 'scanning', but got '$STATUS'"
398 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
399 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
401 local POSITION0=$($SHOW_SCRUB |
402 awk '/^last_checkpoint_position/ {print $2}')
403 POSITION0=$((POSITION0 + 1))
405 echo "stop $SINGLEMDS"
406 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
408 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
409 do_facet $SINGLEMDS $LCTL set_param fail_val=3
410 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
411 echo "start $SINGLEMDS without disabling OI scrub"
412 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
413 error "(12) Fail to start MDS!"
415 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
416 [ "$STATUS" == "scanning" ] ||
417 error "(13) Expect 'scanning', but got '$STATUS'"
419 local POSITION1=$($SHOW_SCRUB |
420 awk '/^latest_start_position/ {print $2}')
421 [ $POSITION0 -eq $POSITION1 ] ||
422 error "(14) Expect position: $POSITION0, but got $POSITION1"
424 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
425 do_facet $SINGLEMDS $LCTL set_param fail_val=0
427 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
428 [ "$STATUS" == "completed" ] ||
429 error "(15) Expect 'completed', but got '$STATUS'"
431 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
432 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
434 run_test 6 "OI scrub resumes from last checkpoint"
438 mds_backup_restore || error "(1) Fail to backup/restore!"
440 echo "start $SINGLEMDS with disabling OI scrub"
441 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
442 error "(2) Fail to start MDS!"
444 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
445 [ "$STATUS" == "init" ] ||
446 error "(3) Expect 'init', but got '$STATUS'"
448 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
449 [ "$FLAGS" == "inconsistent" ] ||
450 error "(4) Expect 'inconsistent', but got '$FLAGS'"
452 mount_client $MOUNT || error "(5) Fail to start client!"
454 do_facet $SINGLEMDS \
455 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
456 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
457 do_facet $SINGLEMDS $LCTL set_param fail_val=3
458 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
459 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
460 error "(6) File diff failed unexpected!"
462 stat $DIR/$tdir/${tfile}300 ||
463 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
465 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
466 [ "$STATUS" == "scanning" ] ||
467 error "(8) Expect 'scanning', but got '$STATUS'"
469 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
470 [ "$FLAGS" == "inconsistent,auto" ] ||
471 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
473 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
474 do_facet $SINGLEMDS $LCTL set_param fail_val=0
476 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
477 [ "$STATUS" == "completed" ] ||
478 error "(10) Expect 'completed', but got '$STATUS'"
480 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
481 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
483 run_test 7 "System is available during OI scrub scanning"
487 mds_backup_restore || error "(1) Fail to backup/restore!"
489 echo "start $SINGLEMDS with disabling OI scrub"
490 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
491 error "(2) Fail to start MDS!"
493 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
494 [ "$STATUS" == "init" ] ||
495 error "(3) Expect 'init', but got '$STATUS'"
497 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
498 [ "$FLAGS" == "inconsistent" ] ||
499 error "(4) Expect 'inconsistent', but got '$FLAGS'"
501 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
502 do_facet $SINGLEMDS $LCTL set_param fail_val=1
503 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
504 $START_SCRUB || error "(5) Fail to start OI scrub!"
506 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
507 [ "$STATUS" == "scanning" ] ||
508 error "(6) Expect 'scanning', but got '$STATUS'"
510 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
512 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
513 [ "$STATUS" == "stopped" ] ||
514 error "(8) Expect 'stopped', but got '$STATUS'"
516 $START_SCRUB || error "(9) Fail to start OI scrub!"
518 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
519 [ "$STATUS" == "scanning" ] ||
520 error "(10) Expect 'scanning', but got '$STATUS'"
522 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
523 do_facet $SINGLEMDS $LCTL set_param fail_val=0
525 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
526 [ "$STATUS" == "completed" ] ||
527 error "(11) Expect 'completed', but got '$STATUS'"
529 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
530 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
532 run_test 8 "Control OI scrub manually"
535 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
536 skip "Testing on UP system, the speed may be inaccurate."
541 mds_backup_restore || error "(1) Fail to backup/restore!"
543 echo "start $SINGLEMDS with disabling OI scrub"
544 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
545 error "(2) Fail to start MDS!"
547 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
548 [ "$STATUS" == "init" ] ||
549 error "(3) Expect 'init', but got '$STATUS'"
551 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
552 [ "$FLAGS" == "inconsistent" ] ||
553 error "(4) Expect 'inconsistent', but got '$FLAGS'"
555 local BASE_SPEED1=100
557 # OI scrub should run with full speed under inconsistent case
558 $START_SCRUB -s $BASE_SPEED1 || error "(5) Fail to start OI scrub!"
561 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
562 [ "$STATUS" == "completed" ] ||
563 error "(6) Expect 'completed', but got '$STATUS'"
565 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
566 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
568 # OI scrub should run with limited speed under non-inconsistent case
569 $START_SCRUB -s $BASE_SPEED1 -r || error "(8) Fail to start OI scrub!"
572 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "scanning" ] ||
574 error "(9) Expect 'scanning', but got '$STATUS'"
576 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
578 # Do NOT ignore that there are 1024 pre-fetched items. And there
579 # may be time error, normally it should be less than 2 seconds.
580 # We allow another 20% schedule error.
581 local PRE_FETCHED=1024
583 # MAX_MARGIN = 1.2 = 12 / 10
584 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
585 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
586 [ $SPEED -lt $MAX_SPEED ] ||
587 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
590 local BASE_SPEED2=300
592 do_facet $SINGLEMDS \
593 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
596 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
597 # MIN_MARGIN = 0.8 = 8 / 10
598 local MIN_SPEED=$(((PRE_FETCHED + \
599 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
600 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
601 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
602 [ $SPEED -gt $MIN_SPEED ] ||
603 error "(11) Got speed $SPEED, expected more than $MIN_SPEED"
605 # MAX_MARGIN = 1.2 = 12 / 10
606 MAX_SPEED=$(((PRE_FETCHED + \
607 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
608 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
609 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
610 [ $SPEED -lt $MAX_SPEED ] ||
611 error "(12) Got speed $SPEED, expected less than $MAX_SPEED"
613 do_facet $SINGLEMDS \
614 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
616 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
617 [ "$STATUS" == "completed" ] ||
618 error "(13) Expect 'completed', but got '$STATUS'"
620 run_test 9 "OI scrub speed control"
624 mds_backup_restore || error "(1) Fail to backup/restore!"
626 echo "start $SINGLEMDS with disabling OI scrub"
627 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
628 error "(2) Fail to start MDS!"
630 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
631 [ "$STATUS" == "init" ] ||
632 error "(3) Expect 'init', but got '$STATUS'"
634 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
635 [ "$FLAGS" == "inconsistent" ] ||
636 error "(4) Expect 'inconsistent', but got '$FLAGS'"
638 mount_client $MOUNT || error "(5) Fail to start client!"
640 do_facet $SINGLEMDS \
641 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
642 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
643 do_facet $SINGLEMDS $LCTL set_param fail_val=1
644 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
645 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
646 error "(6) File diff failed unexpected!"
648 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "scanning" ] ||
650 error "(7) Expect 'scanning', but got '$STATUS'"
652 umount_client $MOUNT || error "(8) Fail to stop client!"
654 echo "stop $SINGLEMDS"
655 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
657 echo "start $SINGLEMDS with disabling OI scrub"
658 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
659 error "(10) Fail to start MDS!"
661 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
662 [ "$STATUS" == "paused" ] ||
663 error "(11) Expect 'paused', but got '$STATUS'"
665 echo "stop $SINGLEMDS"
666 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
668 echo "start $SINGLEMDS without disabling OI scrub"
669 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
670 error "(13) Fail to start MDS!"
672 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
673 [ "$STATUS" == "scanning" ] ||
674 error "(14) Expect 'scanning', but got '$STATUS'"
676 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
677 do_facet $SINGLEMDS $LCTL set_param fail_val=0
679 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
680 [ "$STATUS" == "completed" ] ||
681 error "(15) Expect 'completed', but got '$STATUS'"
683 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
684 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
686 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
688 # test_10b is obsolete, it will be coverded by related sanity-lfsck tests.
691 mds_backup_restore || error "(1) Fail to backup/restore!"
693 echo "start $SINGLEMDS with disabling OI scrub"
694 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
695 error "(2) Fail to start MDS!"
697 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
698 [ "$STATUS" == "init" ] ||
699 error "(3) Expect 'init', but got '$STATUS'"
701 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
702 [ "$FLAGS" == "inconsistent" ] ||
703 error "(4) Expect 'inconsistent', but got '$FLAGS'"
705 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
706 do_facet $SINGLEMDS $LCTL set_param fail_val=3
707 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
709 $START_SCRUB || error "(5) Fail to start OI scrub!"
711 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
712 [ "$STATUS" == "scanning" ] ||
713 error "(6) Expect 'scanning', but got '$STATUS'"
715 echo "stop $SINGLEMDS"
716 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
718 echo "start $SINGLEMDS with disabling OI scrub"
719 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
720 error "(8) Fail to start MDS!"
722 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
723 [ "$STATUS" == "paused" ] ||
724 error "(9) Expect 'paused', but got '$STATUS'"
726 echo "stop $SINGLEMDS"
727 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
729 echo "start $SINGLEMDS without disabling OI scrub"
730 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
731 error "(11) Fail to start MDS!"
733 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
734 [ "$STATUS" == "scanning" ] ||
735 error "(12) Expect 'scanning', but got '$STATUS'"
737 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
738 do_facet $SINGLEMDS $LCTL set_param fail_val=0
740 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
741 [ "$STATUS" == "completed" ] ||
742 error "(13) Expect 'completed', but got '$STATUS'"
744 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
745 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
747 #run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
753 formatall > /dev/null
758 local tname=`date +%s`
759 rm -rf $MOUNT/$tname > /dev/null
760 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
762 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
765 do_facet $SINGLEMDS $LCTL clear
766 start_full_debug_logging
767 # reset OI scrub start point by force
768 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
770 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
771 [ "$STATUS" == "completed" ] ||
772 error "(4) Expect 'completed', but got '$STATUS'"
774 # OI scrub should skip the new created objects for the first accessing
775 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
776 # notice we're creating a new llog for every OST on every startup
777 # new features can make this even less stable, so we only check
778 # that the number of skipped files is less than 2x the number of files
779 local MAXIMUM=$((CREATED * 2))
780 local MINIMUM=$((CREATED + 1)) # files + directory
781 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
782 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
784 # reset OI scrub start point by force
785 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
787 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
788 [ "$STATUS" == "completed" ] ||
789 error "(7) Expect 'completed', but got '$STATUS'"
791 # OI scrub should skip the new created object only once
792 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
793 [ $SKIPPED -eq 0 ] ||
794 error "(8) Expect 0 objects skipped, but got $SKIPPED"
796 stop_full_debug_logging
797 restore_mount $MOUNT || error "(9) Fail to start client!"
798 rm -rf $MOUNT/$tname > /dev/null
800 run_test 11 "OI scrub skips the new created objects only once"
802 # restore MDS/OST size
803 MDSSIZE=${SAVED_MDSSIZE}
804 OSTSIZE=${SAVED_OSTSIZE}
806 # cleanup the system at last