3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 [ "${MDSFSTYPE:-$FSTYPE}" != "ldiskfs" ] &&
21 skip "test OI scrub only for ldiskfs" && exit 0
22 require_dsh_mds || exit 0
24 SAVED_MDSSIZE=${MDSSIZE}
25 SAVED_OSTSIZE=${OSTSIZE}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
31 check_and_setup_lustre
34 MDT_DEV="${FSNAME}-MDT0000"
35 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
36 SHOW_SCRUB="do_facet $SINGLEMDS \
37 $LCTL get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
47 echo "preparing... ${nfiles} files will be created."
49 cp $LUSTRE/tests/*.sh $DIR/$tdir/
50 [[ $nfiles -gt 0 ]] && { createmany -o $DIR/$tdir/$tfile $nfiles ||
51 error "createmany failed"; }
54 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
55 echo "stop $SINGLEMDS"
56 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
61 echo "start $SINGLEMDS without disabling OI scrub"
62 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
63 error "(1) Fail to start MDS!"
65 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
66 [ "$STATUS" == "init" ] ||
67 error "(2) Expect 'init', but got '$STATUS'"
69 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
70 [ -z "$FLAGS" ] || error "(3) Expect empty flags, but got '$FLAGS'"
72 mount_client $MOUNT || error "(4) Fail to start client!"
74 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
75 error "(5) File diff failed unexpected!"
77 run_test 0 "Do not auto trigger OI scrub for non-backup/restore case"
81 mds_remove_ois || error "(1) Fail to remove/recreate!"
83 echo "start $SINGLEMDS without disabling OI scrub"
84 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
85 error "(2) Fail to start MDS!"
88 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
89 [ "$STATUS" == "completed" ] ||
90 error "(3) Expect 'completed', but got '$STATUS'"
92 mount_client $MOUNT || error "(4) Fail to start client!"
94 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
95 error "(5) File diff failed unexpected!"
97 run_test 1a "Trigger OI scrub when MDT mounts for OI files remove/recreate case"
101 mds_backup_restore || error "(1) Fail to backup/restore!"
103 echo "start $SINGLEMDS without disabling OI scrub"
104 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
105 error "(2) Fail to start MDS!"
108 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
109 [ "$STATUS" == "completed" ] ||
110 error "(3) Expect 'completed', but got '$STATUS'"
112 mount_client $MOUNT || error "(4) Fail to start client!"
114 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
115 error "(5) File diff failed unexpected!"
117 run_test 2 "Trigger OI scrub when MDT mounts for backup/restore case"
121 mds_backup_restore || error "(1) Fail to backup/restore!"
123 echo "start $SINGLEMDS with disabling OI scrub"
124 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
125 error "(2) Fail to start MDS!"
127 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
128 [ "$STATUS" == "init" ] ||
129 error "(3) Expect 'init', but got '$STATUS'"
131 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
132 [ "$FLAGS" == "inconsistent" ] ||
133 error "(4) Expect 'inconsistent', but got '$FLAGS'"
137 run_test 3 "Do not trigger OI scrub when MDT mounts if 'noscrub' specified"
141 mds_backup_restore || error "(1) Fail to backup/restore!"
143 echo "start $SINGLEMDS with disabling OI scrub"
144 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
145 error "(2) Fail to start MDS!"
147 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
148 [ "$STATUS" == "init" ] ||
149 error "(3) Expect 'init', but got '$STATUS'"
151 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
152 [ "$FLAGS" == "inconsistent" ] ||
153 error "(4) Expect 'inconsistent', but got '$FLAGS'"
155 mount_client $MOUNT || error "(5) Fail to start client!"
157 do_facet $SINGLEMDS \
158 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
159 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
160 error "(6) File diff failed unexpected!"
163 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
164 [ "$STATUS" == "completed" ] ||
165 error "(7) Expect 'completed', but got '$STATUS'"
167 run_test 4 "Trigger OI scrub automatically if inconsistent OI mapping was found"
171 mds_backup_restore || error "(1) Fail to backup/restore!"
173 echo "start $SINGLEMDS with disabling OI scrub"
174 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
175 error "(2) Fail to start MDS!"
177 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
178 [ "$STATUS" == "init" ] ||
179 error "(3) Expect 'init', but got '$STATUS'"
181 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
182 [ "$FLAGS" == "inconsistent" ] ||
183 error "(4) Expect 'inconsistent', but got '$FLAGS'"
185 mount_client $MOUNT || error "(5) Fail to start client!"
187 do_facet $SINGLEMDS \
188 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
189 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
190 do_facet $SINGLEMDS $LCTL set_param fail_val=3
191 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
192 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
193 error "(6) File diff failed unexpected!"
195 umount_client $MOUNT || error "(7) Fail to stop client!"
197 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
198 [ "$STATUS" == "scanning" ] ||
199 error "(8) Expect 'scanning', but got '$STATUS'"
201 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
202 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
204 echo "stop $SINGLEMDS"
205 stop $SINGLEMDS > /dev/null || error "(9) Fail to stop MDS!"
207 echo "start $SINGLEMDS with disabling OI scrub"
208 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
209 error "(10) Fail to start MDS!"
211 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
212 [ "$STATUS" == "crashed" ] ||
213 error "(11) Expect 'crashed', but got '$STATUS'"
215 echo "stop $SINGLEMDS"
216 stop $SINGLEMDS > /dev/null || error "(12) Fail to stop MDS!"
218 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
219 do_facet $SINGLEMDS $LCTL set_param fail_val=3
220 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
221 echo "start $SINGLEMDS without disabling OI scrub"
222 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
223 error "(13) Fail to start MDS!"
225 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
226 [ "$STATUS" == "scanning" ] ||
227 error "(14) Expect 'scanning', but got '$STATUS'"
229 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
230 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
232 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
233 [ "$STATUS" == "failed" ] ||
234 error "(15) Expect 'failed', but got '$STATUS'"
236 mount_client $MOUNT || error "(16) Fail to start client!"
238 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
239 do_facet $SINGLEMDS $LCTL set_param fail_val=3
240 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
241 stat $DIR/$tdir/${tfile}1000 ||
242 error "(17) Fail to stat $DIR/$tdir/${tfile}1000!"
244 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
245 [ "$STATUS" == "scanning" ] ||
246 error "(18) Expect 'scanning', but got '$STATUS'"
248 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
249 do_facet $SINGLEMDS $LCTL set_param fail_val=0
251 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
252 [ "$STATUS" == "completed" ] ||
253 error "(19) Expect 'completed', but got '$STATUS'"
255 FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
256 [ -z "$FLAGS" ] || error "(20) Expect empty flags, but got '$FLAGS'"
258 run_test 5 "OI scrub state machine"
262 mds_backup_restore || error "(1) Fail to backup/restore!"
264 echo "start $SINGLEMDS with disabling OI scrub"
265 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
266 error "(2) Fail to start MDS!"
268 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
269 [ "$STATUS" == "init" ] ||
270 error "(3) Expect 'init', but got '$STATUS'"
272 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
273 [ "$FLAGS" == "inconsistent" ] ||
274 error "(4) Expect 'inconsistent', but got '$FLAGS'"
276 mount_client $MOUNT || error "(5) Fail to start client!"
278 do_facet $SINGLEMDS \
279 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
280 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
281 do_facet $SINGLEMDS $LCTL set_param fail_val=3
282 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
283 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
284 error "(6) File diff failed unexpected!"
286 # Fail the OI scrub to guarantee there is at least on checkpoint
287 #define OBD_FAIL_OSD_SCRUB_FATAL 0x192
288 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000192
290 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
291 [ "$STATUS" == "failed" ] ||
292 error "(7) Expect 'failed', but got '$STATUS'"
294 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
295 do_facet $SINGLEMDS $LCTL set_param fail_val=3
296 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
297 stat $DIR/$tdir/${tfile}800 ||
298 error "(8) Fail to stat $DIR/$tdir/${tfile}800!"
300 umount_client $MOUNT || error "(9) Fail to stop client!"
302 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
303 [ "$STATUS" == "scanning" ] ||
304 error "(10) Expect 'scanning', but got '$STATUS'"
306 #define OBD_FAIL_OSD_SCRUB_CRASH 0x191
307 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80000191
309 local POSITION0=$($SHOW_SCRUB | sed -n '11'p | awk '{print $2}')
310 POSITION0=$((POSITION0 + 1))
312 echo "stop $SINGLEMDS"
313 stop $SINGLEMDS > /dev/null || error "(11) Fail to stop MDS!"
315 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
316 do_facet $SINGLEMDS $LCTL set_param fail_val=3
317 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
318 echo "start $SINGLEMDS without disabling OI scrub"
319 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
320 error "(12) Fail to start MDS!"
322 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
323 [ "$STATUS" == "scanning" ] ||
324 error "(13) Expect 'scanning', but got '$STATUS'"
326 local POSITION1=$($SHOW_SCRUB | sed -n '10'p |awk '{print $2}')
327 [ $POSITION0 -eq $POSITION1 ] ||
328 error "(14) Expect position: $POSITION0, but got $POSITION1"
330 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
331 do_facet $SINGLEMDS $LCTL set_param fail_val=0
333 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
334 [ "$STATUS" == "completed" ] ||
335 error "(15) Expect 'completed', but got '$STATUS'"
337 FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
338 [ -z "$FLAGS" ] || error "(16) Expect empty flags, but got '$FLAGS'"
340 run_test 6 "OI scrub resumes from last checkpoint"
344 mds_backup_restore || error "(1) Fail to backup/restore!"
346 echo "start $SINGLEMDS with disabling OI scrub"
347 start $SINGLEMDS $MDT_DEVNAME $MDS_MOUNT_OPTS,noscrub > /dev/null ||
348 error "(2) Fail to start MDS!"
350 local STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
351 [ "$STATUS" == "init" ] ||
352 error "(3) Expect 'init', but got '$STATUS'"
354 local FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
355 [ "$FLAGS" == "inconsistent" ] ||
356 error "(4) Expect 'inconsistent', but got '$FLAGS'"
358 mount_client $MOUNT || error "(5) Fail to start client!"
360 do_facet $SINGLEMDS \
361 $LCTL set_param -n osd-ldiskfs.${MDT_DEV}.auto_scrub 1
362 #define OBD_FAIL_OSD_SCRUB_DELAY 0x190
363 do_facet $SINGLEMDS $LCTL set_param fail_val=3
364 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x190
365 diff -q $LUSTRE/tests/test-framework.sh $DIR/$tdir/test-framework.sh ||
366 error "(6) File diff failed unexpected!"
368 stat $DIR/$tdir/${tfile}300 ||
369 error "(7) Fail to stat $DIR/$tdir/${tfile}300!"
371 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
372 [ "$STATUS" == "scanning" ] ||
373 error "(8) Expect 'scanning', but got '$STATUS'"
375 FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
376 [ "$FLAGS" == "inconsistent,auto" ] ||
377 error "(9) Expect 'inconsistent,auto', but got '$FLAGS'"
379 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
380 do_facet $SINGLEMDS $LCTL set_param fail_val=0
382 STATUS=$($SHOW_SCRUB | sed -n '4'p | awk '{print $2}')
383 [ "$STATUS" == "completed" ] ||
384 error "(10) Expect 'completed', but got '$STATUS'"
386 FLAGS=$($SHOW_SCRUB | sed -n '5'p | awk '{print $2}')
387 [ -z "$FLAGS" ] || error "(11) Expect empty flags, but got '$FLAGS'"
389 run_test 7 "System is available during OI scrub scanning"
391 # restore the ${facet}_MKFS_OPTS variables
392 for facet in MGS MDS OST; do
393 opts=SAVED_${facet}_MKFS_OPTS
394 if [[ -n ${!opts} ]]; then
395 eval ${facet}_MKFS_OPTS=\"${!opts}\"
399 # restore MDS/OST size
400 MDSSIZE=${SAVED_MDSSIZE}
401 OSTSIZE=${SAVED_OSTSIZE}
403 # cleanup the system at last
406 complete $(basename $0) $SECONDS