3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
33 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
35 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
36 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
39 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
42 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.1) ]] &&
43 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 15"
47 MDT_DEV="${FSNAME}-MDT0000"
48 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
49 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
50 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
51 SHOW_SCRUB="do_facet $SINGLEMDS \
52 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
53 MOUNT_OPTS_SCRUB="-o user_xattr"
54 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
64 echo "preparing... ${nfiles} files will be created."
66 cp $LUSTRE/tests/*.sh $DIR/$tdir/
67 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
68 error "createmany failed"; }
71 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
72 echo "stop $SINGLEMDS"
73 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
78 echo "start $SINGLEMDS without disabling OI scrub"
79 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
80 error "(1) Fail to start MDS!"
82 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
83 [ "$STATUS" == "init" ] ||
84 error "(2) Expect 'init', but got '$STATUS'"
86 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
87 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
89 mount_client $MOUNT || error "(4) Fail to start client!"
91 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
92 error "(5) File diff failed unexpected!"
94 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
98 echo "start $SINGLEMDS without disabling OI scrub"
99 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
100 error "(1) Fail to start MDS!"
102 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
103 [ "$STATUS" == "init" ] ||
104 error "(2) Expect 'init', but got '$STATUS'"
106 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
107 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
109 mount_client $MOUNT || error "(4) Fail to start client!"
111 #define OBD_FAIL_OSD_FID_MAPPING 0x193
112 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
113 # update .lustre OI mapping
115 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
117 umount_client $MOUNT || error "(5) Fail to stop client!"
119 echo "stop $SINGLEMDS"
120 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
122 echo "start $SINGLEMDS with disabling OI scrub"
123 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
124 error "(7) Fail to start MDS!"
126 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
127 [ "$STATUS" == "init" ] ||
128 error "(8) Expect 'init', but got '$STATUS'"
130 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
131 [ "$FLAGS" == "inconsistent" ] ||
132 error "(9) Expect 'inconsistent', but got '$FLAGS'"
134 run_test 1a "Auto trigger initial OI scrub when server mounts"
138 mds_remove_ois || error "(1) Fail to remove/recreate!"
140 echo "start $SINGLEMDS without disabling OI scrub"
141 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
142 error "(2) Fail to start MDS!"
145 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
146 [ "$STATUS" == "completed" ] ||
147 error "(3) Expect 'completed', but got '$STATUS'"
149 mount_client $MOUNT || error "(4) Fail to start client!"
151 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
152 error "(5) File diff failed unexpected!"
154 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
159 # OI files to be removed:
162 # idx 2: oi.16.{2,4,8,16,32}
163 # idx 3: oi.16.{3,9,27}
164 # idx 5: oi.16.{5,25}
165 # idx 7: oi.16.{7,49}
166 for index in 0 1 2 3 5 7; do
168 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
170 echo "start $SINGLEMDS with disabling OI scrub"
171 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
172 /dev/null || error "(2) Fail to start MDS!"
174 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
175 [ "$FLAGS" == "recreated" ] ||
176 error "(3) Expect 'recreated', but got '$FLAGS'"
178 $START_SCRUB || error "(4) Fail to start OI scrub!"
180 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
181 [ "$STATUS" == "completed" ] ||
182 error "(5) Expect 'completed', but got '$STATUS'"
184 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
186 error "(6) Expect empty flags, but got '$FLAGS'"
189 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
193 mds_backup_restore || error "(1) Fail to backup/restore!"
195 echo "start $SINGLEMDS without disabling OI scrub"
196 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
197 error "(2) Fail to start MDS!"
200 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
201 [ "$STATUS" == "completed" ] ||
202 error "(3) Expect 'completed', but got '$STATUS'"
204 mount_client $MOUNT || error "(4) Fail to start client!"
206 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
207 error "(5) File diff failed unexpected!"
209 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
213 mds_backup_restore || error "(1) Fail to backup/restore!"
215 echo "start $SINGLEMDS with disabling OI scrub"
216 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
217 error "(2) Fail to start MDS!"
219 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
220 [ "$STATUS" == "init" ] ||
221 error "(3) Expect 'init', but got '$STATUS'"
223 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
224 [ "$FLAGS" == "inconsistent" ] ||
225 error "(4) Expect 'inconsistent', but got '$FLAGS'"
229 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
233 mds_backup_restore || error "(1) Fail to backup/restore!"
235 echo "start $SINGLEMDS with disabling OI scrub"
236 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
237 error "(2) Fail to start MDS!"
239 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
240 [ "$STATUS" == "init" ] ||
241 error "(3) Expect 'init', but got '$STATUS'"
243 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
244 [ "$FLAGS" == "inconsistent" ] ||
245 error "(4) Expect 'inconsistent', but got '$FLAGS'"
247 mount_client $MOUNT || error "(5) Fail to start client!"
249 do_facet $SINGLEMDS \
250 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
251 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
252 error "(6) File diff failed unexpected!"
255 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
256 [ "$STATUS" == "completed" ] ||
257 error "(7) Expect 'completed', but got '$STATUS'"
259 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
263 mds_backup_restore || error "(1) Fail to backup/restore!"
265 echo "start $SINGLEMDS with disabling OI scrub"
266 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
267 error "(2) Fail to start MDS!"
269 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
270 [ "$STATUS" == "init" ] ||
271 error "(3) Expect 'init', but got '$STATUS'"
273 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
274 [ "$FLAGS" == "inconsistent" ] ||
275 error "(4) Expect 'inconsistent', but got '$FLAGS'"
277 mount_client $MOUNT || error "(5) Fail to start client!"
279 do_facet $SINGLEMDS \
280 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
281 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
282 do_facet $SINGLEMDS $LCTL set_param fail_val=3
283 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
284 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
285 error "(6) File diff failed unexpected!"
287 umount_client $MOUNT || error "(7) Fail to stop client!"
289 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
290 [ "$STATUS" == "scanning" ] ||
291 error "(8) Expect 'scanning', but got '$STATUS'"
293 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
294 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
296 echo "stop $SINGLEMDS"
297 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
299 echo "start $SINGLEMDS with disabling OI scrub"
300 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
301 error "(10) Fail to start MDS!"
303 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
304 [ "$STATUS" == "crashed" ] ||
305 error "(11) Expect 'crashed', but got '$STATUS'"
307 echo "stop $SINGLEMDS"
308 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
310 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
311 do_facet $SINGLEMDS $LCTL set_param fail_val=3
312 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
313 echo "start $SINGLEMDS without disabling OI scrub"
314 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
315 error "(13) Fail to start MDS!"
317 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
318 [ "$STATUS" == "scanning" ] ||
319 error "(14) Expect 'scanning', but got '$STATUS'"
321 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
322 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
324 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
325 [ "$STATUS" == "failed" ] ||
326 error "(15) Expect 'failed', but got '$STATUS'"
328 mount_client $MOUNT || error "(16) Fail to start client!"
330 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
331 do_facet $SINGLEMDS $LCTL set_param fail_val=3
332 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
333 stat $DIR/$tdir/${tfile}1000 ||
334 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
336 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
337 [ "$STATUS" == "scanning" ] ||
338 error "(18) Expect 'scanning', but got '$STATUS'"
340 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
341 do_facet $SINGLEMDS $LCTL set_param fail_val=0
343 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
344 [ "$STATUS" == "completed" ] ||
345 error "(19) Expect 'completed', but got '$STATUS'"
347 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
348 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
350 run_test 5 "OI scrub state machine"
354 mds_backup_restore || error "(1) Fail to backup/restore!"
356 echo "start $SINGLEMDS with disabling OI scrub"
357 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
358 error "(2) Fail to start MDS!"
360 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
361 [ "$STATUS" == "init" ] ||
362 error "(3) Expect 'init', but got '$STATUS'"
364 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
365 [ "$FLAGS" == "inconsistent" ] ||
366 error "(4) Expect 'inconsistent', but got '$FLAGS'"
368 mount_client $MOUNT || error "(5) Fail to start client!"
370 do_facet $SINGLEMDS \
371 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
372 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
373 do_facet $SINGLEMDS $LCTL set_param fail_val=3
374 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
375 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
376 error "(6) File diff failed unexpected!"
378 # Sleep 5 sec to guarantee at least one object processed by OI scrub
380 # Fail the OI scrub to guarantee there is at least one checkpoint
381 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
382 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
384 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
385 [ "$STATUS" == "failed" ] ||
386 error "(7) Expect 'failed', but got '$STATUS'"
388 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
389 do_facet $SINGLEMDS $LCTL set_param fail_val=3
390 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
391 # stat will re-trigger OI scrub
392 stat $DIR/$tdir/${tfile}800 ||
393 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
395 umount_client $MOUNT || error "(9) Fail to stop client!"
397 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
398 [ "$STATUS" == "scanning" ] ||
399 error "(10) Expect 'scanning', but got '$STATUS'"
401 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
402 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
404 local POSITION0=$($SHOW_SCRUB |
405 awk '/^last_checkpoint_position/ {print $2}')
406 POSITION0=$((POSITION0 + 1))
408 echo "stop $SINGLEMDS"
409 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
411 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
412 do_facet $SINGLEMDS $LCTL set_param fail_val=3
413 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
414 echo "start $SINGLEMDS without disabling OI scrub"
415 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
416 error "(12) Fail to start MDS!"
418 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
419 [ "$STATUS" == "scanning" ] ||
420 error "(13) Expect 'scanning', but got '$STATUS'"
422 local POSITION1=$($SHOW_SCRUB |
423 awk '/^latest_start_position/ {print $2}')
424 [ $POSITION0 -eq $POSITION1 ] ||
425 error "(14) Expect position: $POSITION0, but got $POSITION1"
427 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
428 do_facet $SINGLEMDS $LCTL set_param fail_val=0
430 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
431 [ "$STATUS" == "completed" ] ||
432 error "(15) Expect 'completed', but got '$STATUS'"
434 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
435 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
437 run_test 6 "OI scrub resumes from last checkpoint"
441 mds_backup_restore || error "(1) Fail to backup/restore!"
443 echo "start $SINGLEMDS with disabling OI scrub"
444 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
445 error "(2) Fail to start MDS!"
447 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
448 [ "$STATUS" == "init" ] ||
449 error "(3) Expect 'init', but got '$STATUS'"
451 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
452 [ "$FLAGS" == "inconsistent" ] ||
453 error "(4) Expect 'inconsistent', but got '$FLAGS'"
455 mount_client $MOUNT || error "(5) Fail to start client!"
457 do_facet $SINGLEMDS \
458 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
459 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
460 do_facet $SINGLEMDS $LCTL set_param fail_val=3
461 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
462 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
463 error "(6) File diff failed unexpected!"
465 stat $DIR/$tdir/${tfile}300 ||
466 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
468 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
469 [ "$STATUS" == "scanning" ] ||
470 error "(8) Expect 'scanning', but got '$STATUS'"
472 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
473 [ "$FLAGS" == "inconsistent,auto" ] ||
474 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
476 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
477 do_facet $SINGLEMDS $LCTL set_param fail_val=0
479 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
480 [ "$STATUS" == "completed" ] ||
481 error "(10) Expect 'completed', but got '$STATUS'"
483 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
484 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
486 run_test 7 "System is available during OI scrub scanning"
490 mds_backup_restore || error "(1) Fail to backup/restore!"
492 echo "start $SINGLEMDS with disabling OI scrub"
493 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
494 error "(2) Fail to start MDS!"
496 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
497 [ "$STATUS" == "init" ] ||
498 error "(3) Expect 'init', but got '$STATUS'"
500 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
501 [ "$FLAGS" == "inconsistent" ] ||
502 error "(4) Expect 'inconsistent', but got '$FLAGS'"
504 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
505 do_facet $SINGLEMDS $LCTL set_param fail_val=3
506 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
507 $START_SCRUB || error "(5) Fail to start OI scrub!"
509 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
510 [ "$STATUS" == "scanning" ] ||
511 error "(6) Expect 'scanning', but got '$STATUS'"
513 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
515 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
516 [ "$STATUS" == "stopped" ] ||
517 error "(8) Expect 'stopped', but got '$STATUS'"
519 $START_SCRUB || error "(9) Fail to start OI scrub!"
521 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
522 [ "$STATUS" == "scanning" ] ||
523 error "(10) Expect 'scanning', but got '$STATUS'"
525 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
526 do_facet $SINGLEMDS $LCTL set_param fail_val=0
528 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
529 [ "$STATUS" == "completed" ] ||
530 error "(11) Expect 'completed', but got '$STATUS'"
532 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
533 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
535 run_test 8 "Control OI scrub manually"
538 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
539 skip "Testing on UP system, the speed may be inaccurate."
544 mds_backup_restore || error "(1) Fail to backup/restore!"
546 echo "start $SINGLEMDS with disabling OI scrub"
547 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
548 error "(2) Fail to start MDS!"
550 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
551 [ "$STATUS" == "init" ] ||
552 error "(3) Expect 'init', but got '$STATUS'"
554 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
555 [ "$FLAGS" == "inconsistent" ] ||
556 error "(4) Expect 'inconsistent', but got '$FLAGS'"
558 local BASE_SPEED1=100
560 # OI scrub should run with full speed under inconsistent case
561 $START_SCRUB -s $BASE_SPEED1 || error "(5) Fail to start OI scrub!"
564 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
565 [ "$STATUS" == "completed" ] ||
566 error "(6) Expect 'completed', but got '$STATUS'"
568 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
569 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
571 # OI scrub should run with limited speed under non-inconsistent case
572 $START_SCRUB -s $BASE_SPEED1 -r || error "(8) Fail to start OI scrub!"
575 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
576 [ "$STATUS" == "scanning" ] ||
577 error "(9) Expect 'scanning', but got '$STATUS'"
579 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
581 # Do NOT ignore that there are 1024 pre-fetched items. And there
582 # may be time error, normally it should be less than 2 seconds.
583 # We allow another 20% schedule error.
584 local PRE_FETCHED=1024
586 # MAX_MARGIN = 1.2 = 12 / 10
587 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
588 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
589 [ $SPEED -lt $MAX_SPEED ] ||
590 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
593 local BASE_SPEED2=300
595 do_facet $SINGLEMDS \
596 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
599 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
600 # MIN_MARGIN = 0.8 = 8 / 10
601 local MIN_SPEED=$(((PRE_FETCHED + \
602 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
603 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
604 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
605 [ $SPEED -gt $MIN_SPEED ] ||
606 error "(11) Got speed $SPEED, expected more than $MIN_SPEED"
608 # MAX_MARGIN = 1.2 = 12 / 10
609 MAX_SPEED=$(((PRE_FETCHED + \
610 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
611 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
612 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
613 [ $SPEED -lt $MAX_SPEED ] ||
614 error "(12) Got speed $SPEED, expected less than $MAX_SPEED"
616 do_facet $SINGLEMDS \
617 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
619 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
620 [ "$STATUS" == "completed" ] ||
621 error "(13) Expect 'completed', but got '$STATUS'"
623 run_test 9 "OI scrub speed control"
627 mds_backup_restore || error "(1) Fail to backup/restore!"
629 echo "start $SINGLEMDS with disabling OI scrub"
630 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
631 error "(2) Fail to start MDS!"
633 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
634 [ "$STATUS" == "init" ] ||
635 error "(3) Expect 'init', but got '$STATUS'"
637 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
638 [ "$FLAGS" == "inconsistent" ] ||
639 error "(4) Expect 'inconsistent', but got '$FLAGS'"
641 mount_client $MOUNT || error "(5) Fail to start client!"
643 do_facet $SINGLEMDS \
644 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
645 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
646 do_facet $SINGLEMDS $LCTL set_param fail_val=3
647 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
648 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
649 error "(6) File diff failed unexpected!"
651 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
652 [ "$STATUS" == "scanning" ] ||
653 error "(7) Expect 'scanning', but got '$STATUS'"
655 umount_client $MOUNT || error "(8) Fail to stop client!"
657 echo "stop $SINGLEMDS"
658 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
660 echo "start $SINGLEMDS with disabling OI scrub"
661 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
662 error "(10) Fail to start MDS!"
664 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
665 [ "$STATUS" == "paused" ] ||
666 error "(11) Expect 'paused', but got '$STATUS'"
668 echo "stop $SINGLEMDS"
669 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
671 echo "start $SINGLEMDS without disabling OI scrub"
672 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
673 error "(13) Fail to start MDS!"
675 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
676 [ "$STATUS" == "scanning" ] ||
677 error "(14) Expect 'scanning', but got '$STATUS'"
679 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
680 do_facet $SINGLEMDS $LCTL set_param fail_val=0
682 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
683 [ "$STATUS" == "completed" ] ||
684 error "(15) Expect 'completed', but got '$STATUS'"
686 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
687 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
689 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
692 [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.4.50) ]] &&
693 skip "Need MDS version less than 2.4.50" && return
696 mds_backup_restore || error "(1) Fail to backup/restore!"
698 echo "start $SINGLEMDS with disabling OI scrub"
699 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
700 error "(2) Fail to start MDS!"
702 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
703 [ "$STATUS" == "init" ] ||
704 error "(3) Expect 'init', but got '$STATUS'"
706 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
707 [ "$FLAGS" == "inconsistent" ] ||
708 error "(4) Expect 'inconsistent', but got '$FLAGS'"
710 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
711 do_facet $SINGLEMDS $LCTL set_param fail_val=3
712 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
714 $START_SCRUB || error "(5) Fail to start OI scrub!"
716 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "scanning" ] ||
718 error "(6) Expect 'scanning', but got '$STATUS'"
720 echo "stop $SINGLEMDS"
721 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
723 echo "start $SINGLEMDS with disabling OI scrub"
724 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
725 error "(8) Fail to start MDS!"
727 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
728 [ "$STATUS" == "paused" ] ||
729 error "(9) Expect 'paused', but got '$STATUS'"
731 echo "stop $SINGLEMDS"
732 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
734 echo "start $SINGLEMDS without disabling OI scrub"
735 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
736 error "(11) Fail to start MDS!"
738 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
739 [ "$STATUS" == "scanning" ] ||
740 error "(12) Expect 'scanning', but got '$STATUS'"
742 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
743 do_facet $SINGLEMDS $LCTL set_param fail_val=0
745 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
746 [ "$STATUS" == "completed" ] ||
747 error "(13) Expect 'completed', but got '$STATUS'"
749 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
750 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
752 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
758 formatall > /dev/null
763 local tname=`date +%s`
764 rm -rf $MOUNT/$tname > /dev/null
765 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
767 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
770 do_facet $SINGLEMDS $LCTL clear
771 start_full_debug_logging
772 # reset OI scrub start point by force
773 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
775 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
776 [ "$STATUS" == "completed" ] ||
777 error "(4) Expect 'completed', but got '$STATUS'"
779 # OI scrub should skip the new created objects for the first accessing
780 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
781 # notice we're creating a new llog for every OST on every startup
782 # new features can make this even less stable, so we only check
783 # that the number of skipped files is less than 2x the number of files
784 local MAXIMUM=$((CREATED * 2))
785 local MINIMUM=$((CREATED + 1)) # files + directory
786 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
787 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
789 # reset OI scrub start point by force
790 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
792 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
793 [ "$STATUS" == "completed" ] ||
794 error "(7) Expect 'completed', but got '$STATUS'"
796 # OI scrub should skip the new created object only once
797 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
798 [ $SKIPPED -eq 0 ] ||
799 error "(8) Expect 0 objects skipped, but got $SKIPPED"
801 stop_full_debug_logging
802 restore_mount $MOUNT || error "(9) Fail to start client!"
803 rm -rf $MOUNT/$tname > /dev/null
805 run_test 11 "OI scrub skips the new created objects only once"
809 mds_backup_restore || error "(1) Fail to backup/restore!"
811 echo "starting MDTs with OI scrub disabled"
812 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
813 error "(2) Fail to start MDS!"
815 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
816 [ "$STATUS" == "init" ] ||
817 error "(3) Expect 'init', but got '$STATUS'"
819 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
820 [ "$FLAGS" == "inconsistent" ] ||
821 error "(4) Expect 'inconsistent', but got '$FLAGS'"
823 # run under dryrun mode
824 $START_SCRUB -n on || error "(5) Fail to start OI scrub!"
827 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
828 [ "$STATUS" == "completed" ] ||
829 error "(6) Expect 'completed', but got '$STATUS'"
831 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
832 [ "$FLAGS" == "inconsistent" ] ||
833 error "(7) Expect 'inconsistent', but got '$FLAGS'"
835 local PARAMS=$($SHOW_SCRUB | awk '/^param/ { print $2 }')
836 [ "$PARAMS" == "dryrun" ] ||
837 error "(8) Expect 'dryrun', but got '$PARAMS'"
839 local REPAIRED=$($SHOW_SCRUB | awk '/^updated/ { print $2 }')
840 [ $REPAIRED -lt 20 ] &&
841 error "(9) Expect at least 20 updated, but got '$REPAIRED'"
843 # run under dryrun mode again
844 $START_SCRUB -n on || error "(10) Fail to start OI scrub!"
847 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
848 [ "$STATUS" == "completed" ] ||
849 error "(11) Expect 'completed', but got '$STATUS'"
851 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
852 [ "$FLAGS" == "inconsistent" ] ||
853 error "(12) Expect 'inconsistent', but got '$FLAGS'"
855 PARAMS=$($SHOW_SCRUB | awk '/^param/ { print $2 }')
856 [ "$PARAMS" == "dryrun" ] ||
857 error "(13) Expect 'dryrun', but got '$PARAMS'"
859 REPAIRED=$($SHOW_SCRUB | awk '/^updated/ { print $2 }')
860 [ $REPAIRED -lt 20 ] &&
861 error "(14) Expect at least 20 updated, but got '$REPAIRED'"
863 # run under normal mode
864 $START_SCRUB -n off || error "(15) Fail to start OI scrub!"
867 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
868 [ "$STATUS" == "completed" ] ||
869 error "(16) Expect 'completed', but got '$STATUS'"
871 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
872 [ -z "$FLAGS" ] || error "(17) Expect empty flags, but got '$FLAGS'"
874 PARAMS=$($SHOW_SCRUB | awk '/^param/ { print $2 }')
875 [ -z "$PARAMS" ] || error "(18) Expect empty param, but got '$PARAMS'"
877 REPAIRED=$($SHOW_SCRUB | awk '/^updated/ { print $2 }')
878 [ $REPAIRED -lt 20 ] &&
879 error "(19) Expect at least 20 updated, but got '$REPAIRED'"
881 # run under normal mode again
882 $START_SCRUB -n off || error "(20) Fail to start OI scrub!"
885 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
886 [ "$STATUS" == "completed" ] ||
887 error "(21) Expect 'completed', but got '$STATUS'"
889 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
890 [ -z "$FLAGS" ] || error "(22) Expect empty flags, but got '$FLAGS'"
892 PARAMS=$($SHOW_SCRUB | awk '/^param/ { print $2 }')
893 [ -z "$PARAMS" ] || error "(23) Expect empty param, but got '$PARAMS'"
895 REPAIRED=$($SHOW_SCRUB | awk '/^updated/ { print $2 }')
896 [ $REPAIRED -eq 0 ] ||
897 error "(24) Expect 0 updated, but got '$REPAIRED'"
899 run_test 15 "Dryrun mode OI scrub"
901 # restore MDS/OST size
902 MDSSIZE=${SAVED_MDSSIZE}
903 OSTSIZE=${SAVED_OSTSIZE}
905 # cleanup the system at last