3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
33 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
35 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
36 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
39 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
44 MDT_DEV="${FSNAME}-MDT0000"
45 OST_DEV="${FSNAME}-OST0000"
46 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
47 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
48 START_SCRUB_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV}"
49 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
50 SHOW_SCRUB="do_facet $SINGLEMDS \
51 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
52 SHOW_SCRUB_ON_OST="do_facet ost1 \
53 $LCTL get_param -n osd-ldiskfs.${OST_DEV}.oi_scrub"
54 MOUNT_OPTS_SCRUB="-o user_xattr"
55 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
65 echo "preparing... ${nfiles} files will be created."
67 cp $LUSTRE/tests/*.sh $DIR/$tdir/
68 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
69 error "createmany failed"; }
72 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
73 echo "stop $SINGLEMDS"
74 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
79 echo "start $SINGLEMDS without disabling OI scrub"
80 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
81 error "(1) Fail to start MDS!"
83 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
84 [ "$STATUS" == "init" ] ||
85 error "(2) Expect 'init', but got '$STATUS'"
87 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
88 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
90 mount_client $MOUNT || error "(4) Fail to start client!"
92 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
93 error "(5) File diff failed unexpected!"
95 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
99 echo "start $SINGLEMDS without disabling OI scrub"
100 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
101 error "(1) Fail to start MDS!"
103 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
104 [ "$STATUS" == "init" ] ||
105 error "(2) Expect 'init', but got '$STATUS'"
107 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
108 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
110 mount_client $MOUNT || error "(4) Fail to start client!"
112 #define OBD_FAIL_OSD_FID_MAPPING 0x193
113 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
114 # update .lustre OI mapping
116 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
118 umount_client $MOUNT || error "(5) Fail to stop client!"
120 echo "stop $SINGLEMDS"
121 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
123 echo "start $SINGLEMDS with disabling OI scrub"
124 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
125 error "(7) Fail to start MDS!"
127 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
128 [ "$STATUS" == "init" ] ||
129 error "(8) Expect 'init', but got '$STATUS'"
131 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
132 [ "$FLAGS" == "inconsistent" ] ||
133 error "(9) Expect 'inconsistent', but got '$FLAGS'"
135 run_test 1a "Auto trigger initial OI scrub when server mounts"
139 mds_remove_ois || error "(1) Fail to remove/recreate!"
141 echo "start $SINGLEMDS without disabling OI scrub"
142 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
143 error "(2) Fail to start MDS!"
146 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
147 [ "$STATUS" == "completed" ] ||
148 error "(3) Expect 'completed', but got '$STATUS'"
150 mount_client $MOUNT || error "(4) Fail to start client!"
152 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
153 error "(5) File diff failed unexpected!"
155 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
160 # OI files to be removed:
163 # idx 2: oi.16.{2,4,8,16,32}
164 # idx 3: oi.16.{3,9,27}
165 # idx 5: oi.16.{5,25}
166 # idx 7: oi.16.{7,49}
167 for index in 0 1 2 3 5 7; do
169 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
171 echo "start $SINGLEMDS with disabling OI scrub"
172 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
173 /dev/null || error "(2) Fail to start MDS!"
175 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
176 [ "$FLAGS" == "recreated" ] ||
177 error "(3) Expect 'recreated', but got '$FLAGS'"
179 $START_SCRUB || error "(4) Fail to start OI scrub!"
181 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
182 [ "$STATUS" == "completed" ] ||
183 error "(5) Expect 'completed', but got '$STATUS'"
185 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
187 error "(6) Expect empty flags, but got '$FLAGS'"
190 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
194 mds_backup_restore || error "(1) Fail to backup/restore!"
196 echo "start $SINGLEMDS without disabling OI scrub"
197 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
198 error "(2) Fail to start MDS!"
201 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
202 [ "$STATUS" == "completed" ] ||
203 error "(3) Expect 'completed', but got '$STATUS'"
205 mount_client $MOUNT || error "(4) Fail to start client!"
207 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
208 error "(5) File diff failed unexpected!"
210 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
214 mds_backup_restore || error "(1) Fail to backup/restore!"
216 echo "start $SINGLEMDS with disabling OI scrub"
217 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
218 error "(2) Fail to start MDS!"
220 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
221 [ "$STATUS" == "init" ] ||
222 error "(3) Expect 'init', but got '$STATUS'"
224 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
225 [ "$FLAGS" == "inconsistent" ] ||
226 error "(4) Expect 'inconsistent', but got '$FLAGS'"
230 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
234 mds_backup_restore || error "(1) Fail to backup/restore!"
236 echo "start $SINGLEMDS with disabling OI scrub"
237 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
238 error "(2) Fail to start MDS!"
240 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
241 [ "$STATUS" == "init" ] ||
242 error "(3) Expect 'init', but got '$STATUS'"
244 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
245 [ "$FLAGS" == "inconsistent" ] ||
246 error "(4) Expect 'inconsistent', but got '$FLAGS'"
248 mount_client $MOUNT || error "(5) Fail to start client!"
250 do_facet $SINGLEMDS \
251 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
252 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
253 error "(6) File diff failed unexpected!"
256 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
257 [ "$STATUS" == "completed" ] ||
258 error "(7) Expect 'completed', but got '$STATUS'"
260 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
264 mds_backup_restore || error "(1) Fail to backup/restore!"
266 echo "start $SINGLEMDS with disabling OI scrub"
267 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
268 error "(2) Fail to start MDS!"
270 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
271 [ "$STATUS" == "init" ] ||
272 error "(3) Expect 'init', but got '$STATUS'"
274 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
275 [ "$FLAGS" == "inconsistent" ] ||
276 error "(4) Expect 'inconsistent', but got '$FLAGS'"
278 mount_client $MOUNT || error "(5) Fail to start client!"
280 do_facet $SINGLEMDS \
281 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
282 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
283 do_facet $SINGLEMDS $LCTL set_param fail_val=3
284 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
285 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
286 error "(6) File diff failed unexpected!"
288 umount_client $MOUNT || error "(7) Fail to stop client!"
290 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
291 [ "$STATUS" == "scanning" ] ||
292 error "(8) Expect 'scanning', but got '$STATUS'"
294 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
295 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
297 echo "stop $SINGLEMDS"
298 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
300 echo "start $SINGLEMDS with disabling OI scrub"
301 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
302 error "(10) Fail to start MDS!"
304 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
305 [ "$STATUS" == "crashed" ] ||
306 error "(11) Expect 'crashed', but got '$STATUS'"
308 echo "stop $SINGLEMDS"
309 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
311 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
312 do_facet $SINGLEMDS $LCTL set_param fail_val=3
313 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
314 echo "start $SINGLEMDS without disabling OI scrub"
315 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
316 error "(13) Fail to start MDS!"
318 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
319 [ "$STATUS" == "scanning" ] ||
320 error "(14) Expect 'scanning', but got '$STATUS'"
322 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
323 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
325 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
326 [ "$STATUS" == "failed" ] ||
327 error "(15) Expect 'failed', but got '$STATUS'"
329 mount_client $MOUNT || error "(16) Fail to start client!"
331 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
332 do_facet $SINGLEMDS $LCTL set_param fail_val=3
333 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
334 stat $DIR/$tdir/${tfile}1000 ||
335 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
337 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
338 [ "$STATUS" == "scanning" ] ||
339 error "(18) Expect 'scanning', but got '$STATUS'"
341 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
342 do_facet $SINGLEMDS $LCTL set_param fail_val=0
344 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
345 [ "$STATUS" == "completed" ] ||
346 error "(19) Expect 'completed', but got '$STATUS'"
348 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
349 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
351 run_test 5 "OI scrub state machine"
355 mds_backup_restore || error "(1) Fail to backup/restore!"
357 echo "start $SINGLEMDS with disabling OI scrub"
358 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
359 error "(2) Fail to start MDS!"
361 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
362 [ "$STATUS" == "init" ] ||
363 error "(3) Expect 'init', but got '$STATUS'"
365 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
366 [ "$FLAGS" == "inconsistent" ] ||
367 error "(4) Expect 'inconsistent', but got '$FLAGS'"
369 mount_client $MOUNT || error "(5) Fail to start client!"
371 do_facet $SINGLEMDS \
372 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
373 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
374 do_facet $SINGLEMDS $LCTL set_param fail_val=3
375 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
376 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
377 error "(6) File diff failed unexpected!"
379 # Sleep 5 sec to guarantee at least one object processed by OI scrub
381 # Fail the OI scrub to guarantee there is at least one checkpoint
382 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
383 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
385 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
386 [ "$STATUS" == "failed" ] ||
387 error "(7) Expect 'failed', but got '$STATUS'"
389 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
390 do_facet $SINGLEMDS $LCTL set_param fail_val=3
391 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
392 # stat will re-trigger OI scrub
393 stat $DIR/$tdir/${tfile}800 ||
394 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
396 umount_client $MOUNT || error "(9) Fail to stop client!"
398 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
399 [ "$STATUS" == "scanning" ] ||
400 error "(10) Expect 'scanning', but got '$STATUS'"
402 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
403 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
405 local POSITION0=$($SHOW_SCRUB |
406 awk '/^last_checkpoint_position/ {print $2}')
407 POSITION0=$((POSITION0 + 1))
409 echo "stop $SINGLEMDS"
410 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
412 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
413 do_facet $SINGLEMDS $LCTL set_param fail_val=3
414 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
415 echo "start $SINGLEMDS without disabling OI scrub"
416 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
417 error "(12) Fail to start MDS!"
419 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
420 [ "$STATUS" == "scanning" ] ||
421 error "(13) Expect 'scanning', but got '$STATUS'"
423 local POSITION1=$($SHOW_SCRUB |
424 awk '/^latest_start_position/ {print $2}')
425 [ $POSITION0 -eq $POSITION1 ] ||
426 error "(14) Expect position: $POSITION0, but got $POSITION1"
428 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
429 do_facet $SINGLEMDS $LCTL set_param fail_val=0
431 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
432 [ "$STATUS" == "completed" ] ||
433 error "(15) Expect 'completed', but got '$STATUS'"
435 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
436 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
438 run_test 6 "OI scrub resumes from last checkpoint"
442 mds_backup_restore || error "(1) Fail to backup/restore!"
444 echo "start $SINGLEMDS with disabling OI scrub"
445 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
446 error "(2) Fail to start MDS!"
448 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
449 [ "$STATUS" == "init" ] ||
450 error "(3) Expect 'init', but got '$STATUS'"
452 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
453 [ "$FLAGS" == "inconsistent" ] ||
454 error "(4) Expect 'inconsistent', but got '$FLAGS'"
456 mount_client $MOUNT || error "(5) Fail to start client!"
458 do_facet $SINGLEMDS \
459 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
460 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
461 do_facet $SINGLEMDS $LCTL set_param fail_val=3
462 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
463 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
464 error "(6) File diff failed unexpected!"
466 stat $DIR/$tdir/${tfile}300 ||
467 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
469 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
470 [ "$STATUS" == "scanning" ] ||
471 error "(8) Expect 'scanning', but got '$STATUS'"
473 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
474 [ "$FLAGS" == "inconsistent,auto" ] ||
475 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
477 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
478 do_facet $SINGLEMDS $LCTL set_param fail_val=0
480 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
481 [ "$STATUS" == "completed" ] ||
482 error "(10) Expect 'completed', but got '$STATUS'"
484 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
485 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
487 run_test 7 "System is available during OI scrub scanning"
491 mds_backup_restore || error "(1) Fail to backup/restore!"
493 echo "start $SINGLEMDS with disabling OI scrub"
494 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
495 error "(2) Fail to start MDS!"
497 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
498 [ "$STATUS" == "init" ] ||
499 error "(3) Expect 'init', but got '$STATUS'"
501 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
502 [ "$FLAGS" == "inconsistent" ] ||
503 error "(4) Expect 'inconsistent', but got '$FLAGS'"
505 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
506 do_facet $SINGLEMDS $LCTL set_param fail_val=1
507 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
508 $START_SCRUB || error "(5) Fail to start OI scrub!"
510 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
511 [ "$STATUS" == "scanning" ] ||
512 error "(6) Expect 'scanning', but got '$STATUS'"
514 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
516 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
517 [ "$STATUS" == "stopped" ] ||
518 error "(8) Expect 'stopped', but got '$STATUS'"
520 $START_SCRUB || error "(9) Fail to start OI scrub!"
522 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
523 [ "$STATUS" == "scanning" ] ||
524 error "(10) Expect 'scanning', but got '$STATUS'"
526 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
527 do_facet $SINGLEMDS $LCTL set_param fail_val=0
529 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
530 [ "$STATUS" == "completed" ] ||
531 error "(11) Expect 'completed', but got '$STATUS'"
533 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
534 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
536 run_test 8 "Control OI scrub manually"
539 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
540 skip "Testing on UP system, the speed may be inaccurate."
545 mds_backup_restore || error "(1) Fail to backup/restore!"
547 echo "start $SINGLEMDS with disabling OI scrub"
548 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
549 error "(2) Fail to start MDS!"
551 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
552 [ "$STATUS" == "init" ] ||
553 error "(3) Expect 'init', but got '$STATUS'"
555 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
556 [ "$FLAGS" == "inconsistent" ] ||
557 error "(4) Expect 'inconsistent', but got '$FLAGS'"
559 local BASE_SPEED1=100
561 # OI scrub should run with full speed under inconsistent case
562 $START_SCRUB -s $BASE_SPEED1 || error "(5) Fail to start OI scrub!"
565 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
566 [ "$STATUS" == "completed" ] ||
567 error "(6) Expect 'completed', but got '$STATUS'"
569 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
570 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
572 # OI scrub should run with limited speed under non-inconsistent case
573 $START_SCRUB -s $BASE_SPEED1 -r || error "(8) Fail to start OI scrub!"
576 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
577 [ "$STATUS" == "scanning" ] ||
578 error "(9) Expect 'scanning', but got '$STATUS'"
580 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
582 # Do NOT ignore that there are 1024 pre-fetched items. And there
583 # may be time error, normally it should be less than 2 seconds.
584 # We allow another 20% schedule error.
585 local PRE_FETCHED=1024
587 # MAX_MARGIN = 1.2 = 12 / 10
588 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
589 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
590 [ $SPEED -lt $MAX_SPEED ] ||
591 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
594 local BASE_SPEED2=300
596 do_facet $SINGLEMDS \
597 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
600 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
601 # MIN_MARGIN = 0.8 = 8 / 10
602 local MIN_SPEED=$(((PRE_FETCHED + \
603 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
604 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
605 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
606 [ $SPEED -gt $MIN_SPEED ] ||
607 error "(11) Got speed $SPEED, expected more than $MIN_SPEED"
609 # MAX_MARGIN = 1.2 = 12 / 10
610 MAX_SPEED=$(((PRE_FETCHED + \
611 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
612 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
613 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
614 [ $SPEED -lt $MAX_SPEED ] ||
615 error "(12) Got speed $SPEED, expected less than $MAX_SPEED"
617 do_facet $SINGLEMDS \
618 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
620 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
621 [ "$STATUS" == "completed" ] ||
622 error "(13) Expect 'completed', but got '$STATUS'"
624 run_test 9 "OI scrub speed control"
628 mds_backup_restore || error "(1) Fail to backup/restore!"
630 echo "start $SINGLEMDS with disabling OI scrub"
631 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
632 error "(2) Fail to start MDS!"
634 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
635 [ "$STATUS" == "init" ] ||
636 error "(3) Expect 'init', but got '$STATUS'"
638 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
639 [ "$FLAGS" == "inconsistent" ] ||
640 error "(4) Expect 'inconsistent', but got '$FLAGS'"
642 mount_client $MOUNT || error "(5) Fail to start client!"
644 do_facet $SINGLEMDS \
645 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
646 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
647 do_facet $SINGLEMDS $LCTL set_param fail_val=1
648 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
649 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
650 error "(6) File diff failed unexpected!"
652 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
653 [ "$STATUS" == "scanning" ] ||
654 error "(7) Expect 'scanning', but got '$STATUS'"
656 umount_client $MOUNT || error "(8) Fail to stop client!"
658 echo "stop $SINGLEMDS"
659 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
661 echo "start $SINGLEMDS with disabling OI scrub"
662 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
663 error "(10) Fail to start MDS!"
665 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
666 [ "$STATUS" == "paused" ] ||
667 error "(11) Expect 'paused', but got '$STATUS'"
669 echo "stop $SINGLEMDS"
670 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
672 echo "start $SINGLEMDS without disabling OI scrub"
673 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
674 error "(13) Fail to start MDS!"
676 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
677 [ "$STATUS" == "scanning" ] ||
678 error "(14) Expect 'scanning', but got '$STATUS'"
680 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
681 do_facet $SINGLEMDS $LCTL set_param fail_val=0
683 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
684 [ "$STATUS" == "completed" ] ||
685 error "(15) Expect 'completed', but got '$STATUS'"
687 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
688 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
690 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
692 # test_10b is obsolete, it will be coverded by related sanity-lfsck tests.
695 mds_backup_restore || error "(1) Fail to backup/restore!"
697 echo "start $SINGLEMDS with disabling OI scrub"
698 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
699 error "(2) Fail to start MDS!"
701 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
702 [ "$STATUS" == "init" ] ||
703 error "(3) Expect 'init', but got '$STATUS'"
705 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
706 [ "$FLAGS" == "inconsistent" ] ||
707 error "(4) Expect 'inconsistent', but got '$FLAGS'"
709 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
710 do_facet $SINGLEMDS $LCTL set_param fail_val=3
711 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
713 $START_SCRUB || error "(5) Fail to start OI scrub!"
715 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
716 [ "$STATUS" == "scanning" ] ||
717 error "(6) Expect 'scanning', but got '$STATUS'"
719 echo "stop $SINGLEMDS"
720 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
722 echo "start $SINGLEMDS with disabling OI scrub"
723 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
724 error "(8) Fail to start MDS!"
726 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
727 [ "$STATUS" == "paused" ] ||
728 error "(9) Expect 'paused', but got '$STATUS'"
730 echo "stop $SINGLEMDS"
731 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
733 echo "start $SINGLEMDS without disabling OI scrub"
734 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
735 error "(11) Fail to start MDS!"
737 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
738 [ "$STATUS" == "scanning" ] ||
739 error "(12) Expect 'scanning', but got '$STATUS'"
741 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
742 do_facet $SINGLEMDS $LCTL set_param fail_val=0
744 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
745 [ "$STATUS" == "completed" ] ||
746 error "(13) Expect 'completed', but got '$STATUS'"
748 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
749 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
751 #run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
757 formatall > /dev/null
762 local tname=`date +%s`
763 rm -rf $MOUNT/$tname > /dev/null
764 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
766 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
769 do_facet $SINGLEMDS $LCTL clear
770 start_full_debug_logging
771 # reset OI scrub start point by force
772 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
774 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
775 [ "$STATUS" == "completed" ] ||
776 error "(4) Expect 'completed', but got '$STATUS'"
778 # OI scrub should skip the new created objects for the first accessing
779 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
780 # notice we're creating a new llog for every OST on every startup
781 # new features can make this even less stable, so we only check
782 # that the number of skipped files is less than 2x the number of files
783 local MAXIMUM=$((CREATED * 2))
784 local MINIMUM=$((CREATED + 1)) # files + directory
785 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
786 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
788 # reset OI scrub start point by force
789 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
791 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
792 [ "$STATUS" == "completed" ] ||
793 error "(7) Expect 'completed', but got '$STATUS'"
795 # OI scrub should skip the new created object only once
796 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
797 [ $SKIPPED -eq 0 ] ||
798 error "(8) Expect 0 objects skipped, but got $SKIPPED"
800 stop_full_debug_logging
801 restore_mount $MOUNT || error "(9) Fail to start client!"
802 rm -rf $MOUNT/$tname > /dev/null
804 run_test 11 "OI scrub skips the new created objects only once"
810 formatall > /dev/null
815 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
817 #define OBD_FAIL_OSD_COMPAT_INVALID_ENTRY 0x195
818 do_facet ost1 $LCTL set_param fail_loc=0x195
819 createmany -o $DIR/$tdir/f 1000
826 do_facet ost1 $LCTL set_param fail_loc=0
827 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
828 [ "$STATUS" == "init" ] ||
829 error "(1) Expect 'init', but got '$STATUS'"
831 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
834 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
835 [ "$STATUS" == "completed" ] ||
836 error "(3) Expect 'completed', but got '$STATUS'"
838 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(4) ls should succeed"
840 run_test 12 "OI scrub can rebuild invalid /O entries"
846 formatall > /dev/null
851 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
853 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
854 do_facet ost1 $LCTL set_param fail_loc=0x196
855 createmany -o $DIR/$tdir/f 1000
856 do_facet ost1 $LCTL set_param fail_loc=0
863 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
864 [ "$STATUS" == "init" ] ||
865 error "(1) Expect 'init', but got '$STATUS'"
867 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
869 $START_SCRUB_ON_OST || error "(3) Fail to start OI scrub on OST!"
871 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
872 [ "$STATUS" == "completed" ] ||
873 error "(4) Expect 'completed', but got '$STATUS'"
875 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(5) ls should succeed"
877 run_test 13 "OI scrub can rebuild missed /O entries"
879 # restore MDS/OST size
880 MDSSIZE=${SAVED_MDSSIZE}
881 OSTSIZE=${SAVED_OSTSIZE}
883 # cleanup the system at last