3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 MCREATE=${MCREATE:-mcreate}
23 SAVED_MDSSIZE=${MDSSIZE}
24 SAVED_OSTSIZE=${OSTSIZE}
25 SAVED_OSTCOUNT=${OSTCOUNT}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 # no need too much OSTs, to reduce the format/start/stop overhead
31 [ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
33 # build up a clean test environment.
37 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
38 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
41 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
42 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
44 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
45 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
49 $LCTL set_param debug=+lfsck > /dev/null || true
51 MDT_DEV="${FSNAME}-MDT0000"
52 OST_DEV="${FSNAME}-OST0000"
53 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
54 START_NAMESPACE="do_facet $SINGLEMDS \
55 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
56 START_LAYOUT="do_facet $SINGLEMDS \
57 $LCTL lfsck_start -M ${MDT_DEV} -t layout"
58 START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
59 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
60 SHOW_NAMESPACE="do_facet $SINGLEMDS \
61 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
62 SHOW_LAYOUT="do_facet $SINGLEMDS \
63 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
64 SHOW_LAYOUT_ON_OST="do_facet ost1 \
65 $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
66 MOUNT_OPTS_SCRUB="-o user_xattr"
67 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
76 echo "preparing... $nfiles * $ndirs files will be created $(date)."
77 if [ ! -z $igif ]; then
78 #define OBD_FAIL_FID_IGIF 0x1504
79 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
82 cp $LUSTRE/tests/*.sh $DIR/$tdir/
83 if [ $ndirs -gt 0 ]; then
84 createmany -d $DIR/$tdir/d $ndirs
85 createmany -m $DIR/$tdir/f $ndirs
86 if [ $nfiles -gt 0 ]; then
87 for ((i = 0; i < $ndirs; i++)); do
88 createmany -m $DIR/$tdir/d${i}/f $nfiles > \
89 /dev/null || error "createmany $nfiles"
92 createmany -d $DIR/$tdir/e $ndirs
95 if [ ! -z $igif ]; then
96 touch $DIR/$tdir/dummy
97 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
100 echo "prepared $(date)."
106 #define OBD_FAIL_LFSCK_DELAY1 0x1600
107 do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x1600
108 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
110 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
112 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
113 [ "$STATUS" == "scanning-phase1" ] ||
114 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
116 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
118 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
119 [ "$STATUS" == "stopped" ] ||
120 error "(6) Expect 'stopped', but got '$STATUS'"
122 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
124 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
125 [ "$STATUS" == "scanning-phase1" ] ||
126 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
128 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
129 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
130 mdd.${MDT_DEV}.lfsck_namespace |
131 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
133 error "(9) unexpected status"
136 local repaired=$($SHOW_NAMESPACE |
137 awk '/^updated_phase1/ { print $2 }')
138 [ $repaired -eq 0 ] ||
139 error "(10) Expect nothing to be repaired, but got: $repaired"
141 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
142 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
143 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
144 mdd.${MDT_DEV}.lfsck_namespace |
145 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
147 error "(12) unexpected status"
150 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
151 [ $((scanned1 + 1)) -eq $scanned2 ] ||
152 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
154 echo "stopall, should NOT crash LU-3649"
155 stopall || error "(14) Fail to stopall"
157 run_test 0 "Control LFSCK manually"
160 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
161 skip "OI Scrub not implemented for ZFS" && return
165 #define OBD_FAIL_FID_INDIR 0x1501
166 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
167 touch $DIR/$tdir/dummy
169 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
171 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
172 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
173 mdd.${MDT_DEV}.lfsck_namespace |
174 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
176 error "(4) unexpected status"
179 local repaired=$($SHOW_NAMESPACE |
180 awk '/^updated_phase1/ { print $2 }')
181 [ $repaired -eq 1 ] ||
182 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
184 mount_client $MOUNT || error "(6) Fail to start client!"
186 #define OBD_FAIL_FID_LOOKUP 0x1505
187 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
188 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
190 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
192 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
196 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
197 skip "OI Scrub not implemented for ZFS" && return
201 #define OBD_FAIL_FID_INLMA 0x1502
202 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
203 touch $DIR/$tdir/dummy
205 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
207 #define OBD_FAIL_FID_NOLMA 0x1506
208 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
209 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
210 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
211 mdd.${MDT_DEV}.lfsck_namespace |
212 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
214 error "(4) unexpected status"
217 local repaired=$($SHOW_NAMESPACE |
218 awk '/^updated_phase1/ { print $2 }')
219 [ $repaired -eq 1 ] ||
220 error "(5) Fail to repair missed FID-in-LMA: $repaired"
222 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
223 mount_client $MOUNT || error "(6) Fail to start client!"
225 #define OBD_FAIL_FID_LOOKUP 0x1505
226 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
227 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
229 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
231 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
236 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
237 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
238 touch $DIR/$tdir/dummy
240 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
242 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
243 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
244 mdd.${MDT_DEV}.lfsck_namespace |
245 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
247 error "(4) unexpected status"
250 local repaired=$($SHOW_NAMESPACE |
251 awk '/^updated_phase1/ { print $2 }')
252 [ $repaired -eq 1 ] ||
253 error "(5) Fail to repair crashed linkEA: $repaired"
255 mount_client $MOUNT || error "(6) Fail to start client!"
257 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
258 error "(7) Fail to stat $DIR/$tdir/dummy"
260 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
261 local dummyname=$($LFS fid2path $DIR $dummyfid)
262 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
263 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
265 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
271 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
272 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
273 touch $DIR/$tdir/dummy
275 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
277 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
278 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
279 mdd.${MDT_DEV}.lfsck_namespace |
280 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
282 error "(4) unexpected status"
285 local repaired=$($SHOW_NAMESPACE |
286 awk '/^updated_phase2/ { print $2 }')
287 [ $repaired -eq 1 ] ||
288 error "(5) Fail to repair crashed linkEA: $repaired"
290 mount_client $MOUNT || error "(6) Fail to start client!"
292 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
293 error "(7) Fail to stat $DIR/$tdir/dummy"
295 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
296 local dummyname=$($LFS fid2path $DIR $dummyfid)
297 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
298 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
300 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
306 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
307 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
308 touch $DIR/$tdir/dummy
310 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
312 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
313 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
314 mdd.${MDT_DEV}.lfsck_namespace |
315 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
317 error "(4) unexpected status"
320 local repaired=$($SHOW_NAMESPACE |
321 awk '/^updated_phase2/ { print $2 }')
322 [ $repaired -eq 1 ] ||
323 error "(5) Fail to repair crashed linkEA: $repaired"
325 mount_client $MOUNT || error "(6) Fail to start client!"
327 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
328 error "(7) Fail to stat $DIR/$tdir/dummy"
330 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
331 local dummyname=$($LFS fid2path $DIR $dummyfid)
332 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
333 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
335 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
339 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
340 skip "OI Scrub not implemented for ZFS" && return
343 cleanup_mount $MOUNT || error "(0.1) Fail to stop client!"
344 stop $SINGLEMDS > /dev/null || error "(0.2) Fail to stop MDS!"
346 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
347 echo "start $SINGLEMDS with disabling OI scrub"
348 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
349 error "(2) Fail to start MDS!"
351 #define OBD_FAIL_LFSCK_DELAY2 0x1601
352 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
353 $START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
354 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
355 mdd.${MDT_DEV}.lfsck_namespace |
356 awk '/^flags/ { print \\\$2 }'" "inconsistent" 6 || {
358 error "(5) unexpected status"
361 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
362 [ "$STATUS" == "scanning-phase1" ] ||
363 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
365 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
366 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
367 mdd.${MDT_DEV}.lfsck_namespace |
368 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
370 error "(7) unexpected status"
373 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
374 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
376 local repaired=$($SHOW_NAMESPACE |
377 awk '/^updated_phase1/ { print $2 }')
378 [ $repaired -ge 9 ] ||
379 error "(9) Fail to repair crashed linkEA: $repaired"
381 mount_client $MOUNT || error "(10) Fail to start client!"
383 #define OBD_FAIL_FID_LOOKUP 0x1505
384 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
385 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
386 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
388 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
392 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
393 skip "OI Scrub not implemented for ZFS" && return
396 cleanup_mount $MOUNT || error "(0.1) Fail to stop client!"
397 stop $SINGLEMDS > /dev/null || error "(0.2) Fail to stop MDS!"
399 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
400 echo "start $SINGLEMDS with disabling OI scrub"
401 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
402 error "(2) Fail to start MDS!"
404 #define OBD_FAIL_LFSCK_DELAY2 0x1601
405 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
406 $START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
407 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
408 mdd.${MDT_DEV}.lfsck_namespace |
409 awk '/^flags/ { print \\\$2 }'" "inconsistent,upgrade" 6 || {
411 error "(5) unexpected status"
414 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
415 [ "$STATUS" == "scanning-phase1" ] ||
416 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
418 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
419 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
420 mdd.${MDT_DEV}.lfsck_namespace |
421 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
423 error "(7) unexpected status"
426 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
427 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
429 local repaired=$($SHOW_NAMESPACE |
430 awk '/^updated_phase1/ { print $2 }')
431 [ $repaired -ge 2 ] ||
432 error "(9) Fail to repair crashed linkEA: $repaired"
434 mount_client $MOUNT || error "(10) Fail to start client!"
436 #define OBD_FAIL_FID_LOOKUP 0x1505
437 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
438 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
440 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
442 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
443 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
444 local dummyname=$($LFS fid2path $DIR $dummyfid)
445 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
446 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
448 run_test 5 "LFSCK can handle IGIF object upgrading"
453 #define OBD_FAIL_LFSCK_DELAY1 0x1600
454 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1600
455 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
457 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
458 [ "$STATUS" == "scanning-phase1" ] ||
459 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
461 # Sleep 3 sec to guarantee at least one object processed by LFSCK
463 # Fail the LFSCK to guarantee there is at least one checkpoint
464 #define OBD_FAIL_LFSCK_FATAL1 0x1608
465 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
466 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
467 mdd.${MDT_DEV}.lfsck_namespace |
468 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
470 error "(4) unexpected status"
473 local POS0=$($SHOW_NAMESPACE |
474 awk '/^last_checkpoint_position/ { print $2 }' |
477 #define OBD_FAIL_LFSCK_DELAY1 0x1600
478 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1600
479 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
481 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
482 [ "$STATUS" == "scanning-phase1" ] ||
483 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
485 local POS1=$($SHOW_NAMESPACE |
486 awk '/^latest_start_position/ { print $2 }' |
488 [ $POS0 -lt $POS1 ] ||
489 error "(7) Expect larger than: $POS0, but got $POS1"
491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
492 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
493 mdd.${MDT_DEV}.lfsck_namespace |
494 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
496 error "(8) unexpected status"
499 run_test 6a "LFSCK resumes from last checkpoint (1)"
504 #define OBD_FAIL_LFSCK_DELAY2 0x1601
505 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
506 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
508 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
509 [ "$STATUS" == "scanning-phase1" ] ||
510 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
512 # Sleep 5 sec to guarantee that we are in the directory scanning
514 # Fail the LFSCK to guarantee there is at least one checkpoint
515 #define OBD_FAIL_LFSCK_FATAL2 0x1609
516 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
517 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
518 mdd.${MDT_DEV}.lfsck_namespace |
519 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
521 error "(4) unexpected status"
524 local O_POS0=$($SHOW_NAMESPACE |
525 awk '/^last_checkpoint_position/ { print $2 }' |
528 local D_POS0=$($SHOW_NAMESPACE |
529 awk '/^last_checkpoint_position/ { print $4 }')
531 #define OBD_FAIL_LFSCK_DELAY2 0x1601
532 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
533 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
535 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
536 [ "$STATUS" == "scanning-phase1" ] ||
537 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
539 local O_POS1=$($SHOW_NAMESPACE |
540 awk '/^latest_start_position/ { print $2 }' |
542 local D_POS1=$($SHOW_NAMESPACE |
543 awk '/^latest_start_position/ { print $4 }')
545 if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then
546 [ $O_POS0 -lt $O_POS1 ] ||
547 error "(7.1) $O_POS1 is not larger than $O_POS0"
549 [ $D_POS0 -lt $D_POS1 ] ||
550 error "(7.2) $D_POS1 is not larger than $D_POS0"
553 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
554 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
555 mdd.${MDT_DEV}.lfsck_namespace |
556 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
558 error "(8) unexpected status"
561 run_test 6b "LFSCK resumes from last checkpoint (2)"
568 #define OBD_FAIL_LFSCK_DELAY2 0x1601
569 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
570 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
572 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "scanning-phase1" ] ||
574 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
576 # Sleep 3 sec to guarantee at least one object processed by LFSCK
578 echo "stop $SINGLEMDS"
579 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(5) Fail to start MDS!"
585 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
586 [ "$STATUS" == "scanning-phase1" ] ||
587 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
589 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
590 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
591 mdd.${MDT_DEV}.lfsck_namespace |
592 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
594 error "(7) unexpected status"
597 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
603 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
604 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
605 for ((i = 0; i < 20; i++)); do
606 touch $DIR/$tdir/dummy${i}
609 #define OBD_FAIL_LFSCK_DELAY3 0x1602
610 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1602
611 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
612 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
613 mdd.${MDT_DEV}.lfsck_namespace |
614 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
616 error "(4) unexpected status"
619 echo "stop $SINGLEMDS"
620 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
622 echo "start $SINGLEMDS"
623 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
624 error "(6) Fail to start MDS!"
626 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
627 [ "$STATUS" == "scanning-phase2" ] ||
628 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
630 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
631 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
632 mdd.${MDT_DEV}.lfsck_namespace |
633 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
635 error "(8) unexpected status"
638 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
643 formatall > /dev/null
649 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
650 [ "$STATUS" == "init" ] ||
651 error "(2) Expect 'init', but got '$STATUS'"
653 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
654 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
655 mkdir $DIR/$tdir/crashed
657 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
658 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
659 for ((i = 0; i < 5; i++)); do
660 touch $DIR/$tdir/dummy${i}
663 umount_client $MOUNT || error "(3) Fail to stop client!"
665 #define OBD_FAIL_LFSCK_DELAY2 0x1601
666 do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1601
667 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
669 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
670 [ "$STATUS" == "scanning-phase1" ] ||
671 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
673 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
675 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
676 [ "$STATUS" == "stopped" ] ||
677 error "(7) Expect 'stopped', but got '$STATUS'"
679 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
681 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
682 [ "$STATUS" == "scanning-phase1" ] ||
683 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
685 #define OBD_FAIL_LFSCK_FATAL2 0x1609
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
687 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
688 mdd.${MDT_DEV}.lfsck_namespace |
689 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
691 error "(10) unexpected status"
694 #define OBD_FAIL_LFSCK_DELAY1 0x1600
695 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
696 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
698 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
699 [ "$STATUS" == "scanning-phase1" ] ||
700 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
702 #define OBD_FAIL_LFSCK_CRASH 0x160a
703 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
706 echo "stop $SINGLEMDS"
707 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
709 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
710 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
712 echo "start $SINGLEMDS"
713 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
714 error "(14) Fail to start MDS!"
716 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "crashed" ] ||
718 error "(15) Expect 'crashed', but got '$STATUS'"
720 #define OBD_FAIL_LFSCK_DELAY2 0x1601
721 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
722 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
724 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
725 [ "$STATUS" == "scanning-phase1" ] ||
726 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
728 echo "stop $SINGLEMDS"
729 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
731 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
732 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
734 echo "start $SINGLEMDS"
735 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
736 error "(19) Fail to start MDS!"
738 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
739 [ "$STATUS" == "paused" ] ||
740 error "(20) Expect 'paused', but got '$STATUS'"
742 #define OBD_FAIL_LFSCK_DELAY3 0x1602
743 do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1602
745 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
746 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
747 mdd.${MDT_DEV}.lfsck_namespace |
748 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
750 error "(22) unexpected status"
753 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
754 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
755 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
757 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
758 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
759 mdd.${MDT_DEV}.lfsck_namespace |
760 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
762 error "(24) unexpected status"
765 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
766 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
768 run_test 8 "LFSCK state machine"
771 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
772 skip "Testing on UP system, the speed may be inaccurate."
778 local BASE_SPEED1=100
780 $START_NAMESPACE -r -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
783 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
784 [ "$STATUS" == "scanning-phase1" ] ||
785 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
787 local SPEED=$($SHOW_NAMESPACE |
788 awk '/^average_speed_phase1/ { print $2 }')
790 # There may be time error, normally it should be less than 2 seconds.
791 # We allow another 20% schedule error.
793 # MAX_MARGIN = 1.2 = 12 / 10
794 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
795 RUN_TIME1 * 12 / 10))
796 [ $SPEED -lt $MAX_SPEED ] ||
797 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
800 local BASE_SPEED2=300
802 do_facet $SINGLEMDS \
803 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
806 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
807 # MIN_MARGIN = 0.8 = 8 / 10
808 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
809 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
810 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
811 # Account for slow ZFS performance - LU-4934
812 [ $SPEED -gt $MIN_SPEED ] || [ $(facet_fstype $SINGLEMDS) -eq zfs ] ||
813 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
815 # MAX_MARGIN = 1.2 = 12 / 10
816 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
817 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
818 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
819 [ $SPEED -lt $MAX_SPEED ] ||
820 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
822 do_facet $SINGLEMDS \
823 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
825 wait_update_facet $SINGLEMDS \
826 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
827 awk '/^status/ { print \\\$2 }'" "completed" 30 ||
828 error "(7) Failed to get expected 'completed'"
830 run_test 9a "LFSCK speed control (1)"
833 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
834 skip "Testing on UP system, the speed may be inaccurate."
840 echo "Preparing another 50 * 50 files (with error) at $(date)."
841 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
842 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
843 createmany -d $DIR/$tdir/d 50
844 createmany -m $DIR/$tdir/f 50
845 for ((i = 0; i < 50; i++)); do
846 createmany -m $DIR/$tdir/d${i}/f 50 > /dev/null
849 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
850 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
851 $START_NAMESPACE -r || error "(4) Fail to start LFSCK!"
852 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
853 mdd.${MDT_DEV}.lfsck_namespace |
854 awk '/^status/ { print \\\$2 }'" "stopped" 10 || {
856 error "(5) unexpected status"
859 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
860 echo "Prepared at $(date)."
864 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
867 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
868 [ "$STATUS" == "scanning-phase2" ] ||
869 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
871 local SPEED=$($SHOW_NAMESPACE |
872 awk '/^average_speed_phase2/ { print $2 }')
873 # There may be time error, normally it should be less than 2 seconds.
874 # We allow another 20% schedule error.
876 # MAX_MARGIN = 1.2 = 12 / 10
877 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
878 RUN_TIME1 * 12 / 10))
879 [ $SPEED -lt $MAX_SPEED ] ||
880 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
883 local BASE_SPEED2=150
885 do_facet $SINGLEMDS \
886 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
889 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
890 # MIN_MARGIN = 0.8 = 8 / 10
891 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
892 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
893 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
894 [ $SPEED -gt $MIN_SPEED ] ||[ $(facet_fstype $SINGLEMDS) -eq zfs ] ||
895 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
897 # MAX_MARGIN = 1.2 = 12 / 10
898 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
899 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
900 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
901 [ $SPEED -lt $MAX_SPEED ] ||
902 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
904 do_facet $SINGLEMDS \
905 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
906 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
907 mdd.${MDT_DEV}.lfsck_namespace |
908 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
910 error "(11) unexpected status"
913 run_test 9b "LFSCK speed control (2)"
917 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
918 skip "lookup(..)/linkea on ZFS issue" && return
922 echo "Preparing more files with error at $(date)."
923 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
924 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
926 for ((i = 0; i < 1000; i = $((i+2)))); do
927 mkdir -p $DIR/$tdir/d${i}
928 touch $DIR/$tdir/f${i}
929 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
932 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
933 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
935 for ((i = 1; i < 1000; i = $((i+2)))); do
936 mkdir -p $DIR/$tdir/d${i}
937 touch $DIR/$tdir/f${i}
938 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
941 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
942 echo "Prepared at $(date)."
944 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
947 mount_client $MOUNT || error "(3) Fail to start client!"
949 $START_NAMESPACE -r -s 100 || error "(5) Fail to start LFSCK!"
952 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
953 [ "$STATUS" == "scanning-phase1" ] ||
954 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
956 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
958 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
960 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
962 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
964 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
966 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
968 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
970 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
971 error "(14) Fail to softlink!"
973 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
974 [ "$STATUS" == "scanning-phase1" ] ||
975 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
977 do_facet $SINGLEMDS \
978 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
979 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
980 mdd.${MDT_DEV}.lfsck_namespace |
981 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
983 error "(16) unexpected status"
986 run_test 10 "System is available during LFSCK scanning"
989 ost_remove_lastid() {
992 local rcmd="do_facet ost${ost}"
994 echo "remove LAST_ID on ost${ost}: idx=${idx}"
996 # step 1: local mount
997 mount_fstype ost${ost} || return 1
998 # step 2: remove the specified LAST_ID
999 ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/{LAST_ID,d0/0}
1001 unmount_fstype ost${ost} || return 2
1005 check_mount_and_prep
1006 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1007 createmany -o $DIR/$tdir/f 64 || error "(0) Fail to create 64 files."
1012 ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
1014 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
1015 error "(2) Fail to start ost1"
1017 #define OBD_FAIL_LFSCK_DELAY4 0x160e
1018 do_facet ost1 $LCTL set_param fail_val=3 fail_loc=0x160e
1020 echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
1021 $START_LAYOUT_ON_OST -r || error "(4) Fail to start LFSCK on OST!"
1023 wait_update_facet ost1 "$LCTL get_param -n \
1024 obdfilter.${OST_DEV}.lfsck_layout |
1025 awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
1027 error "(5) unexpected status"
1030 do_facet ost1 $LCTL set_param fail_val=0 fail_loc=0
1032 wait_update_facet ost1 "$LCTL get_param -n \
1033 obdfilter.${OST_DEV}.lfsck_layout |
1034 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1036 error "(6) unexpected status"
1039 echo "the LAST_ID(s) should have been rebuilt"
1040 FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
1041 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
1043 run_test 11a "LFSCK can rebuild lost last_id"
1046 check_mount_and_prep
1047 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1049 echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
1050 #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
1051 do_facet ost1 $LCTL set_param fail_loc=0x160d
1052 createmany -o $DIR/$tdir/f 64
1053 local lastid1=$(do_facet ost1 "lctl get_param -n \
1054 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1055 awk -F: '{ print $2 }')
1057 umount_client $MOUNT
1058 stop ost1 || error "(1) Fail to stop ost1"
1060 #define OBD_FAIL_OST_ENOSPC 0x215
1061 do_facet ost1 $LCTL set_param fail_loc=0x215
1063 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1064 error "(2) Fail to start ost1"
1066 for ((i = 0; i < 60; i++)); do
1067 lastid2=$(do_facet ost1 "lctl get_param -n \
1068 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1069 awk -F: '{ print $2 }')
1070 [ ! -z $lastid2 ] && break;
1074 echo "the on-disk LAST_ID should be smaller than the expected one"
1075 [ $lastid1 -gt $lastid2 ] ||
1076 error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
1078 echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
1079 $START_LAYOUT_ON_OST -r || error "(5) Fail to start LFSCK on OST!"
1081 wait_update_facet ost1 "$LCTL get_param -n \
1082 obdfilter.${OST_DEV}.lfsck_layout |
1083 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1085 error "(6) unexpected status"
1088 stop ost1 || error "(7) Fail to stop ost1"
1090 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1091 error "(8) Fail to start ost1"
1093 echo "the on-disk LAST_ID should have been rebuilt"
1094 wait_update_facet ost1 "$LCTL get_param -n \
1095 obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
1096 awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
1097 $LCTL get_param -n obdfilter.${ost1_svc}.last_id
1098 error "(9) expect lastid1 0x100000000:$lastid1"
1101 do_facet ost1 $LCTL set_param fail_loc=0
1102 stopall || error "(10) Fail to stopall"
1104 run_test 11b "LFSCK can rebuild crashed last_id"
1107 [ $MDSCOUNT -lt 2 ] &&
1108 skip "We need at least 2 MDSes for test_12" && exit 0
1110 check_mount_and_prep
1111 for k in $(seq $MDSCOUNT); do
1112 $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
1113 createmany -o $DIR/$tdir/${k}/f 100 ||
1114 error "(0) Fail to create 100 files."
1117 echo "Start namespace LFSCK on all targets by single command (-s 1)."
1118 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1119 -s 1 -r || error "(2) Fail to start LFSCK on all devices!"
1121 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1122 for k in $(seq $MDSCOUNT); do
1123 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1124 mdd.$(facet_svc mds${k}).lfsck_namespace |
1125 awk '/^status/ { print $2 }')
1126 [ "$STATUS" == "scanning-phase1" ] ||
1127 error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1130 echo "Stop namespace LFSCK on all targets by single lctl command."
1131 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1132 error "(4) Fail to stop LFSCK on all devices!"
1134 echo "All the LFSCK targets should be in 'stopped' status."
1135 for k in $(seq $MDSCOUNT); do
1136 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1137 mdd.$(facet_svc mds${k}).lfsck_namespace |
1138 awk '/^status/ { print $2 }')
1139 [ "$STATUS" == "stopped" ] ||
1140 error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
1143 echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
1144 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1145 -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
1147 echo "All the LFSCK targets should be in 'completed' status."
1148 for k in $(seq $MDSCOUNT); do
1149 wait_update_facet mds${k} "$LCTL get_param -n \
1150 mdd.$(facet_svc mds${k}).lfsck_namespace |
1151 awk '/^status/ { print \\\$2 }'" "completed" 8 ||
1152 error "(7) MDS${k} is not the expected 'completed'"
1155 echo "Start layout LFSCK on all targets by single command (-s 1)."
1156 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1157 -s 1 -r || error "(8) Fail to start LFSCK on all devices!"
1159 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1160 for k in $(seq $MDSCOUNT); do
1161 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1162 mdd.$(facet_svc mds${k}).lfsck_layout |
1163 awk '/^status/ { print $2 }')
1164 [ "$STATUS" == "scanning-phase1" ] ||
1165 error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1168 echo "Stop layout LFSCK on all targets by single lctl command."
1169 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1170 error "(10) Fail to stop LFSCK on all devices!"
1172 echo "All the LFSCK targets should be in 'stopped' status."
1173 for k in $(seq $MDSCOUNT); do
1174 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1175 mdd.$(facet_svc mds${k}).lfsck_layout |
1176 awk '/^status/ { print $2 }')
1177 [ "$STATUS" == "stopped" ] ||
1178 error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
1181 for k in $(seq $OSTCOUNT); do
1182 local STATUS=$(do_facet ost${k} $LCTL get_param -n \
1183 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1184 awk '/^status/ { print $2 }')
1185 [ "$STATUS" == "stopped" ] ||
1186 error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
1189 echo "Re-start layout LFSCK on all targets by single command (-s 0)."
1190 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1191 -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
1193 echo "All the LFSCK targets should be in 'completed' status."
1194 for k in $(seq $MDSCOUNT); do
1195 # The LFSCK status query internal is 30 seconds. For the case
1196 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1197 # time to guarantee the status sync up.
1198 wait_update_facet mds${k} "$LCTL get_param -n \
1199 mdd.$(facet_svc mds${k}).lfsck_layout |
1200 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1201 error "(14) MDS${k} is not the expected 'completed'"
1204 run_test 12 "single command to trigger LFSCK on all devices"
1208 echo "The lmm_oi in layout EA should be consistent with the MDT-object"
1209 echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
1210 echo "MDT-object FID."
1213 check_mount_and_prep
1215 echo "Inject failure stub to simulate bad lmm_oi"
1216 #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
1217 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
1218 createmany -o $DIR/$tdir/f 32
1219 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1221 echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
1222 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1224 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1225 mdd.${MDT_DEV}.lfsck_layout |
1226 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1228 error "(2) unexpected status"
1231 local repaired=$($SHOW_LAYOUT |
1232 awk '/^repaired_others/ { print $2 }')
1233 [ $repaired -eq 32 ] ||
1234 error "(3) Fail to repair crashed lmm_oi: $repaired"
1236 run_test 13 "LFSCK can repair crashed lmm_oi"
1240 echo "The OST-object referenced by the MDT-object should be there;"
1241 echo "otherwise, the LFSCK should re-create the missed OST-object."
1244 check_mount_and_prep
1245 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1247 local count=$(precreated_ost_obj_count 0 0)
1249 echo "Inject failure stub to simulate dangling referenced MDT-object"
1250 #define OBD_FAIL_LFSCK_DANGLING 0x1610
1251 do_facet ost1 $LCTL set_param fail_loc=0x1610
1252 createmany -o $DIR/$tdir/f $((count + 32))
1253 do_facet ost1 $LCTL set_param fail_loc=0
1255 # exhaust other pre-created dangling cases
1256 count=$(precreated_ost_obj_count 0 0)
1257 createmany -o $DIR/$tdir/a $count ||
1258 error "(0) Fail to create $count files."
1260 echo "'ls' should fail because of dangling referenced MDT-object"
1261 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
1263 echo "Trigger layout LFSCK to find out dangling reference"
1264 $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
1266 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1267 mdd.${MDT_DEV}.lfsck_layout |
1268 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1270 error "(3) unexpected status"
1273 local repaired=$($SHOW_LAYOUT |
1274 awk '/^repaired_dangling/ { print $2 }')
1275 [ $repaired -ge 32 ] ||
1276 error "(4) Fail to repair dangling reference: $repaired"
1278 echo "'ls' should fail because it will not repair dangling by default"
1279 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail."
1281 echo "Trigger layout LFSCK to repair dangling reference"
1282 $START_LAYOUT -r -c || error "(6) Fail to start LFSCK for layout!"
1284 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1285 mdd.${MDT_DEV}.lfsck_layout |
1286 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1288 error "(7) unexpected status"
1291 repaired=$($SHOW_LAYOUT |
1292 awk '/^repaired_dangling/ { print $2 }')
1293 [ $repaired -ge 32 ] ||
1294 error "(8) Fail to repair dangling reference: $repaired"
1296 echo "'ls' should success after layout LFSCK repairing"
1297 ls -ail $DIR/$tdir > /dev/null || error "(9) ls should success."
1299 run_test 14 "LFSCK can repair MDT-object with dangling reference"
1303 echo "If the OST-object referenced by the MDT-object back points"
1304 echo "to some non-exist MDT-object, then the LFSCK should repair"
1305 echo "the OST-object to back point to the right MDT-object."
1308 check_mount_and_prep
1309 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1311 echo "Inject failure stub to make the OST-object to back point to"
1312 echo "non-exist MDT-object."
1313 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
1315 do_facet ost1 $LCTL set_param fail_loc=0x1611
1316 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1317 cancel_lru_locks osc
1318 do_facet ost1 $LCTL set_param fail_loc=0
1320 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1321 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1323 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1324 mdd.${MDT_DEV}.lfsck_layout |
1325 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1327 error "(2) unexpected status"
1330 local repaired=$($SHOW_LAYOUT |
1331 awk '/^repaired_unmatched_pair/ { print $2 }')
1332 [ $repaired -eq 1 ] ||
1333 error "(3) Fail to repair unmatched pair: $repaired"
1335 run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
1339 echo "If the OST-object referenced by the MDT-object back points"
1340 echo "to other MDT-object that doesn't recognize the OST-object,"
1341 echo "then the LFSCK should repair it to back point to the right"
1342 echo "MDT-object (the first one)."
1345 check_mount_and_prep
1346 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1347 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1348 cancel_lru_locks osc
1350 echo "Inject failure stub to make the OST-object to back point to"
1351 echo "other MDT-object"
1353 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
1354 do_facet ost1 $LCTL set_param fail_loc=0x1612
1355 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1356 cancel_lru_locks osc
1357 do_facet ost1 $LCTL set_param fail_loc=0
1359 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1360 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1362 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1363 mdd.${MDT_DEV}.lfsck_layout |
1364 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1366 error "(2) unexpected status"
1369 local repaired=$($SHOW_LAYOUT |
1370 awk '/^repaired_unmatched_pair/ { print $2 }')
1371 [ $repaired -eq 1 ] ||
1372 error "(3) Fail to repair unmatched pair: $repaired"
1374 run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
1378 echo "If the OST-object's owner information does not match the owner"
1379 echo "information stored in the MDT-object, then the LFSCK trust the"
1380 echo "MDT-object and update the OST-object's owner information."
1383 check_mount_and_prep
1384 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1385 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1386 cancel_lru_locks osc
1388 echo "Inject failure stub to skip OST-object owner changing"
1389 #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
1390 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
1391 chown 1.1 $DIR/$tdir/f0
1392 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1394 echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
1397 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1399 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1400 mdd.${MDT_DEV}.lfsck_layout |
1401 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1403 error "(2) unexpected status"
1406 local repaired=$($SHOW_LAYOUT |
1407 awk '/^repaired_inconsistent_owner/ { print $2 }')
1408 [ $repaired -eq 1 ] ||
1409 error "(3) Fail to repair inconsistent owner: $repaired"
1411 run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
1415 echo "If more than one MDT-objects reference the same OST-object,"
1416 echo "and the OST-object only recognizes one MDT-object, then the"
1417 echo "LFSCK should create new OST-objects for such non-recognized"
1421 check_mount_and_prep
1422 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1424 echo "Inject failure stub to make two MDT-objects to refernce"
1425 echo "the OST-object"
1427 #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
1428 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0x1614
1430 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1431 cancel_lru_locks osc
1433 createmany -o $DIR/$tdir/f 1
1435 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
1437 cancel_lru_locks mdc
1438 cancel_lru_locks osc
1440 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
1441 local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
1442 [ $size -eq 1048576 ] ||
1443 error "(1) f0 (wrong) size should be 1048576, but got $size"
1445 echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
1448 $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
1450 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1451 mdd.${MDT_DEV}.lfsck_layout |
1452 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1454 error "(3) unexpected status"
1457 local repaired=$($SHOW_LAYOUT |
1458 awk '/^repaired_multiple_referenced/ { print $2 }')
1459 [ $repaired -eq 1 ] ||
1460 error "(4) Fail to repair multiple references: $repaired"
1462 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
1463 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
1464 error "(5) Fail to write f0."
1465 size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
1466 [ $size -eq 1048576 ] ||
1467 error "(6) guard size should be 1048576, but got $size"
1469 run_test 17 "LFSCK can repair multiple references"
1473 echo "The target MDT-object is there, but related stripe information"
1474 echo "is lost or partly lost. The LFSCK should regenerate the missed"
1475 echo "layout EA entries."
1478 check_mount_and_prep
1479 $LFS mkdir -i 0 $DIR/$tdir/a1
1480 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1481 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1483 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1485 $LFS path2fid $DIR/$tdir/a1/f1
1486 $LFS getstripe $DIR/$tdir/a1/f1
1488 if [ $MDSCOUNT -ge 2 ]; then
1489 $LFS mkdir -i 1 $DIR/$tdir/a2
1490 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1491 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1492 $LFS path2fid $DIR/$tdir/a2/f2
1493 $LFS getstripe $DIR/$tdir/a2/f2
1496 cancel_lru_locks osc
1498 echo "Inject failure, to make the MDT-object lost its layout EA"
1499 #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
1500 do_facet mds1 $LCTL set_param fail_loc=0x1615
1501 chown 1.1 $DIR/$tdir/a1/f1
1503 if [ $MDSCOUNT -ge 2 ]; then
1504 do_facet mds2 $LCTL set_param fail_loc=0x1615
1505 chown 1.1 $DIR/$tdir/a2/f2
1511 do_facet mds1 $LCTL set_param fail_loc=0
1512 if [ $MDSCOUNT -ge 2 ]; then
1513 do_facet mds2 $LCTL set_param fail_loc=0
1516 cancel_lru_locks mdc
1517 cancel_lru_locks osc
1519 echo "The file size should be incorrect since layout EA is lost"
1520 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1521 [ "$cur_size" != "$saved_size" ] ||
1522 error "(1) Expect incorrect file1 size"
1524 if [ $MDSCOUNT -ge 2 ]; then
1525 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1526 [ "$cur_size" != "$saved_size" ] ||
1527 error "(2) Expect incorrect file2 size"
1530 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1531 $START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
1533 for k in $(seq $MDSCOUNT); do
1534 # The LFSCK status query internal is 30 seconds. For the case
1535 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1536 # time to guarantee the status sync up.
1537 wait_update_facet mds${k} "$LCTL get_param -n \
1538 mdd.$(facet_svc mds${k}).lfsck_layout |
1539 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1540 error "(4) MDS${k} is not the expected 'completed'"
1543 for k in $(seq $OSTCOUNT); do
1544 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1545 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1546 awk '/^status/ { print $2 }')
1547 [ "$cur_status" == "completed" ] ||
1548 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1551 local repaired=$(do_facet mds1 $LCTL get_param -n \
1552 mdd.$(facet_svc mds1).lfsck_layout |
1553 awk '/^repaired_orphan/ { print $2 }')
1554 [ $repaired -eq 1 ] ||
1555 error "(6.1) Expect 1 fixed on mds1, but got: $repaired"
1557 if [ $MDSCOUNT -ge 2 ]; then
1558 repaired=$(do_facet mds2 $LCTL get_param -n \
1559 mdd.$(facet_svc mds2).lfsck_layout |
1560 awk '/^repaired_orphan/ { print $2 }')
1561 [ $repaired -eq 2 ] ||
1562 error "(6.2) Expect 2 fixed on mds2, but got: $repaired"
1565 $LFS path2fid $DIR/$tdir/a1/f1
1566 $LFS getstripe $DIR/$tdir/a1/f1
1568 if [ $MDSCOUNT -ge 2 ]; then
1569 $LFS path2fid $DIR/$tdir/a2/f2
1570 $LFS getstripe $DIR/$tdir/a2/f2
1573 echo "The file size should be correct after layout LFSCK scanning"
1574 cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1575 [ "$cur_size" == "$saved_size" ] ||
1576 error "(7) Expect file1 size $saved_size, but got $cur_size"
1578 if [ $MDSCOUNT -ge 2 ]; then
1579 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1580 [ "$cur_size" == "$saved_size" ] ||
1581 error "(8) Expect file2 size $saved_size, but got $cur_size"
1584 run_test 18a "Find out orphan OST-object and repair it (1)"
1588 echo "The target MDT-object is lost. The LFSCK should re-create the"
1589 echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
1590 echo "can move it back to normal namespace manually."
1593 check_mount_and_prep
1594 $LFS mkdir -i 0 $DIR/$tdir/a1
1595 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1596 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1597 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1598 local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
1600 $LFS getstripe $DIR/$tdir/a1/f1
1602 if [ $MDSCOUNT -ge 2 ]; then
1603 $LFS mkdir -i 1 $DIR/$tdir/a2
1604 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1605 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1606 fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
1608 $LFS getstripe $DIR/$tdir/a2/f2
1611 cancel_lru_locks osc
1613 echo "Inject failure, to simulate the case of missing the MDT-object"
1614 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1615 do_facet mds1 $LCTL set_param fail_loc=0x1616
1616 rm -f $DIR/$tdir/a1/f1
1618 if [ $MDSCOUNT -ge 2 ]; then
1619 do_facet mds2 $LCTL set_param fail_loc=0x1616
1620 rm -f $DIR/$tdir/a2/f2
1626 do_facet mds1 $LCTL set_param fail_loc=0
1627 if [ $MDSCOUNT -ge 2 ]; then
1628 do_facet mds2 $LCTL set_param fail_loc=0
1631 cancel_lru_locks mdc
1632 cancel_lru_locks osc
1634 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1635 $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
1637 for k in $(seq $MDSCOUNT); do
1638 # The LFSCK status query internal is 30 seconds. For the case
1639 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1640 # time to guarantee the status sync up.
1641 wait_update_facet mds${k} "$LCTL get_param -n \
1642 mdd.$(facet_svc mds${k}).lfsck_layout |
1643 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1644 error "(2) MDS${k} is not the expected 'completed'"
1647 for k in $(seq $OSTCOUNT); do
1648 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1649 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1650 awk '/^status/ { print $2 }')
1651 [ "$cur_status" == "completed" ] ||
1652 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1655 local repaired=$(do_facet mds1 $LCTL get_param -n \
1656 mdd.$(facet_svc mds1).lfsck_layout |
1657 awk '/^repaired_orphan/ { print $2 }')
1658 [ $repaired -eq 1 ] ||
1659 error "(4.1) Expect 1 fixed on mds1, but got: $repaired"
1661 if [ $MDSCOUNT -ge 2 ]; then
1662 repaired=$(do_facet mds2 $LCTL get_param -n \
1663 mdd.$(facet_svc mds2).lfsck_layout |
1664 awk '/^repaired_orphan/ { print $2 }')
1665 [ $repaired -eq 2 ] ||
1666 error "(4.2) Expect 2 fixed on mds2, but got: $repaired"
1669 echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
1670 mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
1671 error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
1673 if [ $MDSCOUNT -ge 2 ]; then
1674 local name=$MOUNT/.lustre/lost+found/MDT0001/R-${fid2}
1675 mv $name $DIR/$tdir/a2/f2 || error "(6) Fail to move $name"
1678 $LFS path2fid $DIR/$tdir/a1/f1
1679 $LFS getstripe $DIR/$tdir/a1/f1
1681 if [ $MDSCOUNT -ge 2 ]; then
1682 $LFS path2fid $DIR/$tdir/a2/f2
1683 $LFS getstripe $DIR/$tdir/a2/f2
1686 echo "The file size should be correct after layout LFSCK scanning"
1687 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1688 [ "$cur_size" == "$saved_size" ] ||
1689 error "(7) Expect file1 size $saved_size, but got $cur_size"
1691 if [ $MDSCOUNT -ge 2 ]; then
1692 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1693 [ "$cur_size" == "$saved_size" ] ||
1694 error "(8) Expect file2 size $saved_size, but got $cur_size"
1697 run_test 18b "Find out orphan OST-object and repair it (2)"
1701 echo "The target MDT-object is lost, and the OST-object FID is missing."
1702 echo "The LFSCK should re-create the MDT-object with new FID under the "
1703 echo "directory .lustre/lost+found/MDTxxxx."
1706 check_mount_and_prep
1707 $LFS mkdir -i 0 $DIR/$tdir/a1
1708 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1710 echo "Inject failure, to simulate the case of missing parent FID"
1711 #define OBD_FAIL_LFSCK_NOPFID 0x1617
1712 do_facet ost1 $LCTL set_param fail_loc=0x1617
1714 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1715 $LFS getstripe $DIR/$tdir/a1/f1
1717 if [ $MDSCOUNT -ge 2 ]; then
1718 $LFS mkdir -i 1 $DIR/$tdir/a2
1719 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1720 do_facet ost2 $LCTL set_param fail_loc=0x1617
1721 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1722 $LFS getstripe $DIR/$tdir/a2/f2
1725 cancel_lru_locks osc
1727 echo "Inject failure, to simulate the case of missing the MDT-object"
1728 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1729 do_facet mds1 $LCTL set_param fail_loc=0x1616
1730 rm -f $DIR/$tdir/a1/f1
1732 if [ $MDSCOUNT -ge 2 ]; then
1733 do_facet mds2 $LCTL set_param fail_loc=0x1616
1734 rm -f $DIR/$tdir/a2/f2
1740 do_facet mds1 $LCTL set_param fail_loc=0
1741 if [ $MDSCOUNT -ge 2 ]; then
1742 do_facet mds2 $LCTL set_param fail_loc=0
1745 cancel_lru_locks mdc
1746 cancel_lru_locks osc
1748 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1749 $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
1751 for k in $(seq $MDSCOUNT); do
1752 # The LFSCK status query internal is 30 seconds. For the case
1753 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1754 # time to guarantee the status sync up.
1755 wait_update_facet mds${k} "$LCTL get_param -n \
1756 mdd.$(facet_svc mds${k}).lfsck_layout |
1757 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1758 error "(2) MDS${k} is not the expected 'completed'"
1761 for k in $(seq $OSTCOUNT); do
1762 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1763 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1764 awk '/^status/ { print $2 }')
1765 [ "$cur_status" == "completed" ] ||
1766 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1769 if [ $MDSCOUNT -ge 2 ]; then
1775 local repaired=$(do_facet mds1 $LCTL get_param -n \
1776 mdd.$(facet_svc mds1).lfsck_layout |
1777 awk '/^repaired_orphan/ { print $2 }')
1778 [ $repaired -eq $expected ] ||
1779 error "(4) Expect $expected fixed on mds1, but got: $repaired"
1781 if [ $MDSCOUNT -ge 2 ]; then
1782 repaired=$(do_facet mds2 $LCTL get_param -n \
1783 mdd.$(facet_svc mds2).lfsck_layout |
1784 awk '/^repaired_orphan/ { print $2 }')
1785 [ $repaired -eq 0 ] ||
1786 error "(5) Expect 0 fixed on mds2, but got: $repaired"
1789 echo "There should be some stub under .lustre/lost+found/MDT0001/"
1790 ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
1791 error "(6) .lustre/lost+found/MDT0001/ should be empty"
1793 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1794 ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
1795 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1797 run_test 18c "Find out orphan OST-object and repair it (3)"
1801 echo "The target MDT-object layout EA slot is occpuied by some new"
1802 echo "created OST-object when repair dangling reference case. Such"
1803 echo "conflict OST-object has never been modified. Then when found"
1804 echo "the orphan OST-object, LFSCK will replace it with the orphan"
1808 check_mount_and_prep
1810 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1811 echo "guard" > $DIR/$tdir/a1/f1
1812 echo "foo" > $DIR/$tdir/a1/f2
1813 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1814 $LFS path2fid $DIR/$tdir/a1/f1
1815 $LFS getstripe $DIR/$tdir/a1/f1
1816 $LFS path2fid $DIR/$tdir/a1/f2
1817 $LFS getstripe $DIR/$tdir/a1/f2
1818 cancel_lru_locks osc
1820 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1821 echo "to reference the same OST-object (which is f1's OST-obejct)."
1822 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1823 echo "dangling reference case, but f2's old OST-object is there."
1826 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1827 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1828 chown 1.1 $DIR/$tdir/a1/f2
1829 rm -f $DIR/$tdir/a1/f1
1832 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1834 echo "stopall to cleanup object cache"
1837 setupall > /dev/null
1839 echo "The file size should be incorrect since dangling referenced"
1840 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1841 [ "$cur_size" != "$saved_size" ] ||
1842 error "(1) Expect incorrect file2 size"
1844 #define OBD_FAIL_LFSCK_DELAY3 0x1602
1845 do_facet $SINGLEMDS $LCTL set_param fail_val=5 fail_loc=0x1602
1847 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1848 $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
1850 wait_update_facet mds1 "$LCTL get_param -n \
1851 mdd.$(facet_svc mds1).lfsck_layout |
1852 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
1853 error "(3.0) MDS1 is not the expected 'scanning-phase2'"
1855 # LU-3469: before osp_sync() is enabled, wait for a while to guarantee
1856 # that former async repair operations have been executed on the OST(s).
1860 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
1862 for k in $(seq $MDSCOUNT); do
1863 # The LFSCK status query internal is 30 seconds. For the case
1864 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1865 # time to guarantee the status sync up.
1866 wait_update_facet mds${k} "$LCTL get_param -n \
1867 mdd.$(facet_svc mds${k}).lfsck_layout |
1868 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1869 error "(3) MDS${k} is not the expected 'completed'"
1872 for k in $(seq $OSTCOUNT); do
1873 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1874 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1875 awk '/^status/ { print $2 }')
1876 [ "$cur_status" == "completed" ] ||
1877 error "(4) OST${k} Expect 'completed', but got '$cur_status'"
1880 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1881 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1882 awk '/^repaired_orphan/ { print $2 }')
1883 [ $repaired -eq 1 ] ||
1884 error "(5) Expect 1 orphan has been fixed, but got: $repaired"
1886 echo "The file size should be correct after layout LFSCK scanning"
1887 cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1888 [ "$cur_size" == "$saved_size" ] ||
1889 error "(6) Expect file2 size $saved_size, but got $cur_size"
1891 echo "The LFSCK should find back the original data."
1892 cat $DIR/$tdir/a1/f2
1893 $LFS path2fid $DIR/$tdir/a1/f2
1894 $LFS getstripe $DIR/$tdir/a1/f2
1896 run_test 18d "Find out orphan OST-object and repair it (4)"
1900 echo "The target MDT-object layout EA slot is occpuied by some new"
1901 echo "created OST-object when repair dangling reference case. Such"
1902 echo "conflict OST-object has been modified by others. To keep the"
1903 echo "new data, the LFSCK will create a new file to refernece this"
1904 echo "old orphan OST-object."
1907 check_mount_and_prep
1909 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1910 echo "guard" > $DIR/$tdir/a1/f1
1911 echo "foo" > $DIR/$tdir/a1/f2
1912 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1913 $LFS path2fid $DIR/$tdir/a1/f1
1914 $LFS getstripe $DIR/$tdir/a1/f1
1915 $LFS path2fid $DIR/$tdir/a1/f2
1916 $LFS getstripe $DIR/$tdir/a1/f2
1917 cancel_lru_locks osc
1919 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1920 echo "to reference the same OST-object (which is f1's OST-obejct)."
1921 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1922 echo "dangling reference case, but f2's old OST-object is there."
1925 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1926 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1927 chown 1.1 $DIR/$tdir/a1/f2
1928 rm -f $DIR/$tdir/a1/f1
1931 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1933 echo "stopall to cleanup object cache"
1936 setupall > /dev/null
1938 echo "The file size should be incorrect since dangling referenced"
1939 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1940 [ "$cur_size" != "$saved_size" ] ||
1941 error "(1) Expect incorrect file2 size"
1943 #define OBD_FAIL_LFSCK_DELAY3 0x1602
1944 do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
1946 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1947 $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
1949 wait_update_facet mds1 "$LCTL get_param -n \
1950 mdd.$(facet_svc mds1).lfsck_layout |
1951 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
1952 error "(3) MDS1 is not the expected 'scanning-phase2'"
1954 # to guarantee all updates are synced.
1958 echo "Write new data to f2 to modify the new created OST-object."
1959 echo "dummy" >> $DIR/$tdir/a1/f2
1961 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
1963 for k in $(seq $MDSCOUNT); do
1964 # The LFSCK status query internal is 30 seconds. For the case
1965 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1966 # time to guarantee the status sync up.
1967 wait_update_facet mds${k} "$LCTL get_param -n \
1968 mdd.$(facet_svc mds${k}).lfsck_layout |
1969 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1970 error "(4) MDS${k} is not the expected 'completed'"
1973 for k in $(seq $OSTCOUNT); do
1974 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1975 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1976 awk '/^status/ { print $2 }')
1977 [ "$cur_status" == "completed" ] ||
1978 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1981 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1982 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1983 awk '/^repaired_orphan/ { print $2 }')
1984 [ $repaired -eq 1 ] ||
1985 error "(6) Expect 1 orphan has been fixed, but got: $repaired"
1987 echo "There should be stub file under .lustre/lost+found/MDT0000/"
1988 local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
1990 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1992 echo "The stub file should keep the original f2 data"
1993 cur_size=$(ls -il $cname | awk '{ print $6 }')
1994 [ "$cur_size" == "$saved_size" ] ||
1995 error "(8) Expect file2 size $saved_size, but got $cur_size"
1998 $LFS path2fid $cname
1999 $LFS getstripe $cname
2001 echo "The f2 should contains new data."
2002 cat $DIR/$tdir/a1/f2
2003 $LFS path2fid $DIR/$tdir/a1/f2
2004 $LFS getstripe $DIR/$tdir/a1/f2
2006 run_test 18e "Find out orphan OST-object and repair it (5)"
2009 check_mount_and_prep
2010 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2012 echo "foo" > $DIR/$tdir/a0
2013 echo "guard" > $DIR/$tdir/a1
2014 cancel_lru_locks osc
2016 echo "Inject failure, then client will offer wrong parent FID when read"
2017 do_facet ost1 $LCTL set_param -n \
2018 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2019 #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
2020 $LCTL set_param fail_loc=0x1619
2022 echo "Read RPC with wrong parent FID should be denied"
2023 cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
2024 $LCTL set_param fail_loc=0
2026 run_test 19a "OST-object inconsistency self detect"
2029 check_mount_and_prep
2030 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2032 echo "Inject failure stub to make the OST-object to back point to"
2033 echo "non-exist MDT-object"
2035 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
2036 do_facet ost1 $LCTL set_param fail_loc=0x1611
2037 echo "foo" > $DIR/$tdir/f0
2038 cancel_lru_locks osc
2039 do_facet ost1 $LCTL set_param fail_loc=0
2041 echo "Nothing should be fixed since self detect and repair is disabled"
2042 local repaired=$(do_facet ost1 $LCTL get_param -n \
2043 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2044 awk '/^repaired/ { print $2 }')
2045 [ $repaired -eq 0 ] ||
2046 error "(1) Expected 0 repaired, but got $repaired"
2048 echo "Read RPC with right parent FID should be accepted,"
2049 echo "and cause parent FID on OST to be fixed"
2051 do_facet ost1 $LCTL set_param -n \
2052 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2053 cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
2055 repaired=$(do_facet ost1 $LCTL get_param -n \
2056 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2057 awk '/^repaired/ { print $2 }')
2058 [ $repaired -eq 1 ] ||
2059 error "(3) Expected 1 repaired, but got $repaired"
2061 run_test 19b "OST-object inconsistency self repair"
2063 $LCTL set_param debug=-lfsck > /dev/null || true
2065 # restore MDS/OST size
2066 MDSSIZE=${SAVED_MDSSIZE}
2067 OSTSIZE=${SAVED_OSTSIZE}
2068 OSTCOUNT=${SAVED_OSTCOUNT}
2070 # cleanup the system at last