3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 SAVED_MDSSIZE=${MDSSIZE}
23 SAVED_OSTSIZE=${OSTSIZE}
24 # use small MDS + OST size to speed formatting time
25 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [ $(facet_fstype $SINGLEMDS) != "ldiskfs" ] &&
33 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
35 [ $(facet_fstype ost1) != "ldiskfs" ] &&
36 skip "test OI scrub only for ldiskfs" && check_and_cleanup_lustre &&
38 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.2.90) ]] &&
39 skip "Need MDS version at least 2.2.90" && check_and_cleanup_lustre &&
42 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.90) ]] &&
43 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 1a"
45 [[ $(lustre_version_code ost1) -lt $(version_code 2.4.50) ]] &&
46 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14"
50 MDT_DEV="${FSNAME}-MDT0000"
51 OST_DEV="${FSNAME}-OST0000"
52 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
53 START_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV}"
54 START_SCRUB_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV}"
55 STOP_SCRUB="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
56 SHOW_SCRUB="do_facet $SINGLEMDS \
57 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
58 SHOW_SCRUB_ON_OST="do_facet ost1 \
59 $LCTL get_param -n osd-ldiskfs.${OST_DEV}.oi_scrub"
60 MOUNT_OPTS_SCRUB="-o user_xattr"
61 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
71 echo "preparing... ${nfiles} files will be created."
73 cp $LUSTRE/tests/*.sh $DIR/$tdir/
74 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
75 error "createmany failed"; }
78 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
79 echo "stop $SINGLEMDS"
80 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
85 echo "start $SINGLEMDS without disabling OI scrub"
86 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
87 error "(1) Fail to start MDS!"
89 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
90 [ "$STATUS" == "init" ] ||
91 error "(2) Expect 'init', but got '$STATUS'"
93 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
94 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
96 mount_client $MOUNT || error "(4) Fail to start client!"
98 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
99 error "(5) File diff failed unexpected!"
101 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
105 echo "start $SINGLEMDS without disabling OI scrub"
106 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
107 error "(1) Fail to start MDS!"
109 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
110 [ "$STATUS" == "init" ] ||
111 error "(2) Expect 'init', but got '$STATUS'"
113 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
114 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
116 mount_client $MOUNT || error "(4) Fail to start client!"
118 #define OBD_FAIL_OSD_FID_MAPPING 0x193
119 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x193
120 # update .lustre OI mapping
122 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
124 umount_client $MOUNT || error "(5) Fail to stop client!"
126 echo "stop $SINGLEMDS"
127 stop $SINGLEMDS > /dev/null || error "(6) Fail to stop MDS!"
129 echo "start $SINGLEMDS with disabling OI scrub"
130 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
131 error "(7) Fail to start MDS!"
133 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
134 [ "$STATUS" == "init" ] ||
135 error "(8) Expect 'init', but got '$STATUS'"
137 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
138 [ "$FLAGS" == "inconsistent" ] ||
139 error "(9) Expect 'inconsistent', but got '$FLAGS'"
141 run_test 1a "Auto trigger initial OI scrub when server mounts"
145 mds_remove_ois || error "(1) Fail to remove/recreate!"
147 echo "start $SINGLEMDS without disabling OI scrub"
148 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
149 error "(2) Fail to start MDS!"
152 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
153 [ "$STATUS" == "completed" ] ||
154 error "(3) Expect 'completed', but got '$STATUS'"
156 mount_client $MOUNT || error "(4) Fail to start client!"
158 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
159 error "(5) File diff failed unexpected!"
161 run_test 1b "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
166 # OI files to be removed:
169 # idx 2: oi.16.{2,4,8,16,32}
170 # idx 3: oi.16.{3,9,27}
171 # idx 5: oi.16.{5,25}
172 # idx 7: oi.16.{7,49}
173 for index in 0 1 2 3 5 7; do
175 mds_remove_ois ${index} || error "(1) Fail to remove/recreate!"
177 echo "start $SINGLEMDS with disabling OI scrub"
178 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > \
179 /dev/null || error "(2) Fail to start MDS!"
181 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
182 [ "$FLAGS" == "recreated" ] ||
183 error "(3) Expect 'recreated', but got '$FLAGS'"
185 $START_SCRUB || error "(4) Fail to start OI scrub!"
187 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
188 [ "$STATUS" == "completed" ] ||
189 error "(5) Expect 'completed', but got '$STATUS'"
191 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
193 error "(6) Expect empty flags, but got '$FLAGS'"
196 run_test 1c "Auto detect kinds of OI file(s) removed/recreated cases"
200 mds_backup_restore || error "(1) Fail to backup/restore!"
202 echo "start $SINGLEMDS without disabling OI scrub"
203 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
204 error "(2) Fail to start MDS!"
207 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
208 [ "$STATUS" == "completed" ] ||
209 error "(3) Expect 'completed', but got '$STATUS'"
211 mount_client $MOUNT || error "(4) Fail to start client!"
213 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
214 error "(5) File diff failed unexpected!"
216 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
220 mds_backup_restore || error "(1) Fail to backup/restore!"
222 echo "start $SINGLEMDS with disabling OI scrub"
223 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
224 error "(2) Fail to start MDS!"
226 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
227 [ "$STATUS" == "init" ] ||
228 error "(3) Expect 'init', but got '$STATUS'"
230 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
231 [ "$FLAGS" == "inconsistent" ] ||
232 error "(4) Expect 'inconsistent', but got '$FLAGS'"
236 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
240 mds_backup_restore || error "(1) Fail to backup/restore!"
242 echo "start $SINGLEMDS with disabling OI scrub"
243 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
244 error "(2) Fail to start MDS!"
246 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
247 [ "$STATUS" == "init" ] ||
248 error "(3) Expect 'init', but got '$STATUS'"
250 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
251 [ "$FLAGS" == "inconsistent" ] ||
252 error "(4) Expect 'inconsistent', but got '$FLAGS'"
254 mount_client $MOUNT || error "(5) Fail to start client!"
256 do_facet $SINGLEMDS \
257 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
258 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
259 error "(6) File diff failed unexpected!"
262 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
263 [ "$STATUS" == "completed" ] ||
264 error "(7) Expect 'completed', but got '$STATUS'"
266 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
270 mds_backup_restore || error "(1) Fail to backup/restore!"
272 echo "start $SINGLEMDS with disabling OI scrub"
273 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
274 error "(2) Fail to start MDS!"
276 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
277 [ "$STATUS" == "init" ] ||
278 error "(3) Expect 'init', but got '$STATUS'"
280 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
281 [ "$FLAGS" == "inconsistent" ] ||
282 error "(4) Expect 'inconsistent', but got '$FLAGS'"
284 mount_client $MOUNT || error "(5) Fail to start client!"
286 do_facet $SINGLEMDS \
287 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
288 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
289 do_facet $SINGLEMDS $LCTL set_param fail_val=3
290 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
291 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
292 error "(6) File diff failed unexpected!"
294 umount_client $MOUNT || error "(7) Fail to stop client!"
296 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
297 [ "$STATUS" == "scanning" ] ||
298 error "(8) Expect 'scanning', but got '$STATUS'"
300 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
301 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
303 echo "stop $SINGLEMDS"
304 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
306 echo "start $SINGLEMDS with disabling OI scrub"
307 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
308 error "(10) Fail to start MDS!"
310 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
311 [ "$STATUS" == "crashed" ] ||
312 error "(11) Expect 'crashed', but got '$STATUS'"
314 echo "stop $SINGLEMDS"
315 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
317 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
318 do_facet $SINGLEMDS $LCTL set_param fail_val=3
319 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
320 echo "start $SINGLEMDS without disabling OI scrub"
321 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
322 error "(13) Fail to start MDS!"
324 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
325 [ "$STATUS" == "scanning" ] ||
326 error "(14) Expect 'scanning', but got '$STATUS'"
328 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
331 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
332 [ "$STATUS" == "failed" ] ||
333 error "(15) Expect 'failed', but got '$STATUS'"
335 mount_client $MOUNT || error "(16) Fail to start client!"
337 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
338 do_facet $SINGLEMDS $LCTL set_param fail_val=3
339 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
340 stat $DIR/$tdir/${tfile}1000 ||
341 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
343 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
344 [ "$STATUS" == "scanning" ] ||
345 error "(18) Expect 'scanning', but got '$STATUS'"
347 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
348 do_facet $SINGLEMDS $LCTL set_param fail_val=0
350 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
351 [ "$STATUS" == "completed" ] ||
352 error "(19) Expect 'completed', but got '$STATUS'"
354 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
355 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
357 run_test 5 "OI scrub state machine"
361 mds_backup_restore || error "(1) Fail to backup/restore!"
363 echo "start $SINGLEMDS with disabling OI scrub"
364 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
365 error "(2) Fail to start MDS!"
367 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
368 [ "$STATUS" == "init" ] ||
369 error "(3) Expect 'init', but got '$STATUS'"
371 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
372 [ "$FLAGS" == "inconsistent" ] ||
373 error "(4) Expect 'inconsistent', but got '$FLAGS'"
375 mount_client $MOUNT || error "(5) Fail to start client!"
377 do_facet $SINGLEMDS \
378 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
379 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
380 do_facet $SINGLEMDS $LCTL set_param fail_val=3
381 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
382 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
383 error "(6) File diff failed unexpected!"
385 # Sleep 5 sec to guarantee at least one object processed by OI scrub
387 # Fail the OI scrub to guarantee there is at least one checkpoint
388 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
389 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
391 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
392 [ "$STATUS" == "failed" ] ||
393 error "(7) Expect 'failed', but got '$STATUS'"
395 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
396 do_facet $SINGLEMDS $LCTL set_param fail_val=3
397 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
398 # stat will re-trigger OI scrub
399 stat $DIR/$tdir/${tfile}800 ||
400 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
402 umount_client $MOUNT || error "(9) Fail to stop client!"
404 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
405 [ "$STATUS" == "scanning" ] ||
406 error "(10) Expect 'scanning', but got '$STATUS'"
408 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
409 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
411 local POSITION0=$($SHOW_SCRUB |
412 awk '/^last_checkpoint_position/ {print $2}')
413 POSITION0=$((POSITION0 + 1))
415 echo "stop $SINGLEMDS"
416 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
418 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
419 do_facet $SINGLEMDS $LCTL set_param fail_val=3
420 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
421 echo "start $SINGLEMDS without disabling OI scrub"
422 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
423 error "(12) Fail to start MDS!"
425 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
426 [ "$STATUS" == "scanning" ] ||
427 error "(13) Expect 'scanning', but got '$STATUS'"
429 local POSITION1=$($SHOW_SCRUB |
430 awk '/^latest_start_position/ {print $2}')
431 [ $POSITION0 -eq $POSITION1 ] ||
432 error "(14) Expect position: $POSITION0, but got $POSITION1"
434 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
435 do_facet $SINGLEMDS $LCTL set_param fail_val=0
437 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
438 [ "$STATUS" == "completed" ] ||
439 error "(15) Expect 'completed', but got '$STATUS'"
441 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
442 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
444 run_test 6 "OI scrub resumes from last checkpoint"
448 mds_backup_restore || error "(1) Fail to backup/restore!"
450 echo "start $SINGLEMDS with disabling OI scrub"
451 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
452 error "(2) Fail to start MDS!"
454 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
455 [ "$STATUS" == "init" ] ||
456 error "(3) Expect 'init', but got '$STATUS'"
458 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
459 [ "$FLAGS" == "inconsistent" ] ||
460 error "(4) Expect 'inconsistent', but got '$FLAGS'"
462 mount_client $MOUNT || error "(5) Fail to start client!"
464 do_facet $SINGLEMDS \
465 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
466 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
467 do_facet $SINGLEMDS $LCTL set_param fail_val=3
468 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
469 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
470 error "(6) File diff failed unexpected!"
472 stat $DIR/$tdir/${tfile}300 ||
473 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
475 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
476 [ "$STATUS" == "scanning" ] ||
477 error "(8) Expect 'scanning', but got '$STATUS'"
479 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
480 [ "$FLAGS" == "inconsistent,auto" ] ||
481 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
483 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
484 do_facet $SINGLEMDS $LCTL set_param fail_val=0
486 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
487 [ "$STATUS" == "completed" ] ||
488 error "(10) Expect 'completed', but got '$STATUS'"
490 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
491 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
493 run_test 7 "System is available during OI scrub scanning"
497 mds_backup_restore || error "(1) Fail to backup/restore!"
499 echo "start $SINGLEMDS with disabling OI scrub"
500 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
501 error "(2) Fail to start MDS!"
503 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
504 [ "$STATUS" == "init" ] ||
505 error "(3) Expect 'init', but got '$STATUS'"
507 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
508 [ "$FLAGS" == "inconsistent" ] ||
509 error "(4) Expect 'inconsistent', but got '$FLAGS'"
511 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
512 do_facet $SINGLEMDS $LCTL set_param fail_val=1
513 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
514 $START_SCRUB || error "(5) Fail to start OI scrub!"
516 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
517 [ "$STATUS" == "scanning" ] ||
518 error "(6) Expect 'scanning', but got '$STATUS'"
520 $STOP_SCRUB || error "(7) Fail to stop OI scrub!"
522 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
523 [ "$STATUS" == "stopped" ] ||
524 error "(8) Expect 'stopped', but got '$STATUS'"
526 $START_SCRUB || error "(9) Fail to start OI scrub!"
528 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
529 [ "$STATUS" == "scanning" ] ||
530 error "(10) Expect 'scanning', but got '$STATUS'"
532 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
533 do_facet $SINGLEMDS $LCTL set_param fail_val=0
535 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
536 [ "$STATUS" == "completed" ] ||
537 error "(11) Expect 'completed', but got '$STATUS'"
539 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
540 [ -z "$FLAGS" ] || error "(12) Expect empty flags, but got '$FLAGS'"
542 run_test 8 "Control OI scrub manually"
545 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
546 skip "Testing on UP system, the speed may be inaccurate."
551 mds_backup_restore || error "(1) Fail to backup/restore!"
553 echo "start $SINGLEMDS with disabling OI scrub"
554 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
555 error "(2) Fail to start MDS!"
557 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
558 [ "$STATUS" == "init" ] ||
559 error "(3) Expect 'init', but got '$STATUS'"
561 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
562 [ "$FLAGS" == "inconsistent" ] ||
563 error "(4) Expect 'inconsistent', but got '$FLAGS'"
565 local BASE_SPEED1=100
567 # OI scrub should run with full speed under inconsistent case
568 $START_SCRUB -s $BASE_SPEED1 || error "(5) Fail to start OI scrub!"
571 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
572 [ "$STATUS" == "completed" ] ||
573 error "(6) Expect 'completed', but got '$STATUS'"
575 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
576 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
578 # OI scrub should run with limited speed under non-inconsistent case
579 $START_SCRUB -s $BASE_SPEED1 -r || error "(8) Fail to start OI scrub!"
582 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
583 [ "$STATUS" == "scanning" ] ||
584 error "(9) Expect 'scanning', but got '$STATUS'"
586 local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
588 # Do NOT ignore that there are 1024 pre-fetched items. And there
589 # may be time error, normally it should be less than 2 seconds.
590 # We allow another 20% schedule error.
591 local PRE_FETCHED=1024
593 # MAX_MARGIN = 1.2 = 12 / 10
594 local MAX_SPEED=$(((PRE_FETCHED + BASE_SPEED1 * \
595 (RUN_TIME1 + TIME_DIFF)) / RUN_TIME1 * 12 / 10))
596 [ $SPEED -lt $MAX_SPEED ] ||
597 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
600 local BASE_SPEED2=300
602 do_facet $SINGLEMDS \
603 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
606 SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
607 # MIN_MARGIN = 0.8 = 8 / 10
608 local MIN_SPEED=$(((PRE_FETCHED + \
609 BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
610 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
611 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
612 [ $SPEED -gt $MIN_SPEED ] ||
613 error "(11) Got speed $SPEED, expected more than $MIN_SPEED"
615 # MAX_MARGIN = 1.2 = 12 / 10
616 MAX_SPEED=$(((PRE_FETCHED + \
617 BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
618 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
619 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
620 [ $SPEED -lt $MAX_SPEED ] ||
621 error "(12) Got speed $SPEED, expected less than $MAX_SPEED"
623 do_facet $SINGLEMDS \
624 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
626 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
627 [ "$STATUS" == "completed" ] ||
628 error "(13) Expect 'completed', but got '$STATUS'"
630 run_test 9 "OI scrub speed control"
634 mds_backup_restore || error "(1) Fail to backup/restore!"
636 echo "start $SINGLEMDS with disabling OI scrub"
637 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
638 error "(2) Fail to start MDS!"
640 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
641 [ "$STATUS" == "init" ] ||
642 error "(3) Expect 'init', but got '$STATUS'"
644 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
645 [ "$FLAGS" == "inconsistent" ] ||
646 error "(4) Expect 'inconsistent', but got '$FLAGS'"
648 mount_client $MOUNT || error "(5) Fail to start client!"
650 do_facet $SINGLEMDS \
651 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
652 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
653 do_facet $SINGLEMDS $LCTL set_param fail_val=1
654 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
655 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
656 error "(6) File diff failed unexpected!"
658 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
659 [ "$STATUS" == "scanning" ] ||
660 error "(7) Expect 'scanning', but got '$STATUS'"
662 umount_client $MOUNT || error "(8) Fail to stop client!"
664 echo "stop $SINGLEMDS"
665 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
667 echo "start $SINGLEMDS with disabling OI scrub"
668 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
669 error "(10) Fail to start MDS!"
671 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
672 [ "$STATUS" == "paused" ] ||
673 error "(11) Expect 'paused', but got '$STATUS'"
675 echo "stop $SINGLEMDS"
676 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
678 echo "start $SINGLEMDS without disabling OI scrub"
679 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
680 error "(13) Fail to start MDS!"
682 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
683 [ "$STATUS" == "scanning" ] ||
684 error "(14) Expect 'scanning', but got '$STATUS'"
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
687 do_facet $SINGLEMDS $LCTL set_param fail_val=0
689 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
690 [ "$STATUS" == "completed" ] ||
691 error "(15) Expect 'completed', but got '$STATUS'"
693 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
694 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
696 run_test 10a "non-stopped OI scrub should auto restarts after MDS remount (1)"
698 # test_10b is obsolete, it will be coverded by related sanity-lfsck tests.
701 mds_backup_restore || error "(1) Fail to backup/restore!"
703 echo "start $SINGLEMDS with disabling OI scrub"
704 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
705 error "(2) Fail to start MDS!"
707 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
708 [ "$STATUS" == "init" ] ||
709 error "(3) Expect 'init', but got '$STATUS'"
711 local FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
712 [ "$FLAGS" == "inconsistent" ] ||
713 error "(4) Expect 'inconsistent', but got '$FLAGS'"
715 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
716 do_facet $SINGLEMDS $LCTL set_param fail_val=3
717 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
719 $START_SCRUB || error "(5) Fail to start OI scrub!"
721 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
722 [ "$STATUS" == "scanning" ] ||
723 error "(6) Expect 'scanning', but got '$STATUS'"
725 echo "stop $SINGLEMDS"
726 stop $SINGLEMDS > /dev/null || error "(7) Fail to stop MDS!"
728 echo "start $SINGLEMDS with disabling OI scrub"
729 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
730 error "(8) Fail to start MDS!"
732 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
733 [ "$STATUS" == "paused" ] ||
734 error "(9) Expect 'paused', but got '$STATUS'"
736 echo "stop $SINGLEMDS"
737 stop $SINGLEMDS > /dev/null || error "(10) Fail to stop MDS!"
739 echo "start $SINGLEMDS without disabling OI scrub"
740 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
741 error "(11) Fail to start MDS!"
743 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
744 [ "$STATUS" == "scanning" ] ||
745 error "(12) Expect 'scanning', but got '$STATUS'"
747 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
748 do_facet $SINGLEMDS $LCTL set_param fail_val=0
750 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
751 [ "$STATUS" == "completed" ] ||
752 error "(13) Expect 'completed', but got '$STATUS'"
754 FLAGS=$($SHOW_SCRUB | awk '/^flags/ { print $2 }')
755 [ -z "$FLAGS" ] || error "(14) Expect empty flags, but got '$FLAGS'"
757 #run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
763 formatall > /dev/null
768 local tname=`date +%s`
769 rm -rf $MOUNT/$tname > /dev/null
770 mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
772 createmany -o $MOUNT/$tname/f $CREATED || error "(2) Fail to create!"
775 do_facet $SINGLEMDS $LCTL clear
776 start_full_debug_logging
777 # reset OI scrub start point by force
778 $START_SCRUB -r || error "(3) Fail to start OI scrub!"
780 local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
781 [ "$STATUS" == "completed" ] ||
782 error "(4) Expect 'completed', but got '$STATUS'"
784 # OI scrub should skip the new created objects for the first accessing
785 local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
786 # notice we're creating a new llog for every OST on every startup
787 # new features can make this even less stable, so we only check
788 # that the number of skipped files is less than 2x the number of files
789 local MAXIMUM=$((CREATED * 2))
790 local MINIMUM=$((CREATED + 1)) # files + directory
791 [ $SKIPPED -ge $MAXIMUM -o $SKIPPED -lt $MINIMUM ] &&
792 error "(5) Expect [ $MINIMUM , $MAXIMUM ) objects skipped, got $SKIPPED"
794 # reset OI scrub start point by force
795 $START_SCRUB -r || error "(6) Fail to start OI scrub!"
797 STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
798 [ "$STATUS" == "completed" ] ||
799 error "(7) Expect 'completed', but got '$STATUS'"
801 # OI scrub should skip the new created object only once
802 SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
803 [ $SKIPPED -eq 0 ] ||
804 error "(8) Expect 0 objects skipped, but got $SKIPPED"
806 stop_full_debug_logging
807 restore_mount $MOUNT || error "(9) Fail to start client!"
808 rm -rf $MOUNT/$tname > /dev/null
810 run_test 11 "OI scrub skips the new created objects only once"
816 formatall > /dev/null
821 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
823 #define OBD_FAIL_OSD_COMPAT_INVALID_ENTRY 0x195
824 do_facet ost1 $LCTL set_param fail_loc=0x195
825 createmany -o $DIR/$tdir/f 1000
832 do_facet ost1 $LCTL set_param fail_loc=0
833 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
834 [ "$STATUS" == "init" ] ||
835 error "(1) Expect 'init', but got '$STATUS'"
837 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
840 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
841 [ "$STATUS" == "completed" ] ||
842 error "(3) Expect 'completed', but got '$STATUS'"
844 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(4) ls should succeed"
846 run_test 12 "OI scrub can rebuild invalid /O entries"
852 formatall > /dev/null
857 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
859 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
860 do_facet ost1 $LCTL set_param fail_loc=0x196
861 createmany -o $DIR/$tdir/f 1000
862 do_facet ost1 $LCTL set_param fail_loc=0
869 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
870 [ "$STATUS" == "init" ] ||
871 error "(1) Expect 'init', but got '$STATUS'"
873 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
875 $START_SCRUB_ON_OST || error "(3) Fail to start OI scrub on OST!"
877 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
878 [ "$STATUS" == "completed" ] ||
879 error "(4) Expect 'completed', but got '$STATUS'"
881 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(5) ls should succeed"
883 run_test 13 "OI scrub can rebuild missed /O entries"
889 formatall > /dev/null
894 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
896 #define OBD_FAIL_OSD_COMPAT_NO_ENTRY 0x196
897 do_facet ost1 $LCTL set_param fail_loc=0x196
898 createmany -o $DIR/$tdir/f 64
899 do_facet ost1 $LCTL set_param fail_loc=0
906 local STATUS=$($SHOW_SCRUB_ON_OST | awk '/^status/ { print $2 }')
907 [ "$STATUS" == "init" ] ||
908 error "(1) Expect 'init', but got '$STATUS'"
910 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(2) ls should fail"
916 run_e2fsck $(facet_host ost1) $(ostdevname 1) "-y" ||
917 error "(3) Fail to run e2fsck error"
922 local LF_REPAIRED=$($SHOW_SCRUB_ON_OST |
923 awk '/^lf_reparied/ { print $2 }')
924 [ $LF_REPAIRED -gt 0 ] ||
925 error "(4) Some entry under /lost+found should be repaired"
927 ls -ail $DIR/$tdir > /dev/null 2>&1 || error "(5) ls should succeed"
929 run_test 14 "OI scrub can repair objects under lost+found"
931 # restore MDS/OST size
932 MDSSIZE=${SAVED_MDSSIZE}
933 OSTSIZE=${SAVED_OSTSIZE}
935 # cleanup the system at last