3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 MCREATE=${MCREATE:-mcreate}
23 SAVED_MDSSIZE=${MDSSIZE}
24 SAVED_OSTSIZE=${OSTSIZE}
25 SAVED_OSTCOUNT=${OSTCOUNT}
26 # use small MDS + OST size to speed formatting time
27 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 # no need too much OSTs, to reduce the format/start/stop overhead
31 [ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
33 # build up a clean test environment.
37 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
38 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
41 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
42 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
44 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
45 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
49 $LCTL set_param debug=+lfsck > /dev/null || true
51 MDT_DEV="${FSNAME}-MDT0000"
52 OST_DEV="${FSNAME}-OST0000"
53 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
54 START_NAMESPACE="do_facet $SINGLEMDS \
55 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
56 START_LAYOUT="do_facet $SINGLEMDS \
57 $LCTL lfsck_start -M ${MDT_DEV} -t layout"
58 START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
59 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
60 SHOW_NAMESPACE="do_facet $SINGLEMDS \
61 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
62 SHOW_LAYOUT="do_facet $SINGLEMDS \
63 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
64 SHOW_LAYOUT_ON_OST="do_facet ost1 \
65 $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
66 MOUNT_OPTS_SCRUB="-o user_xattr"
67 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
76 echo "preparing... $nfiles * $ndirs files will be created $(date)."
77 if [ ! -z $igif ]; then
78 #define OBD_FAIL_FID_IGIF 0x1504
79 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
82 cp $LUSTRE/tests/*.sh $DIR/$tdir/
83 if [ $ndirs -gt 0 ]; then
84 createmany -d $DIR/$tdir/d $ndirs
85 createmany -m $DIR/$tdir/f $ndirs
86 if [ $nfiles -gt 0 ]; then
87 for ((i = 0; i < $ndirs; i++)); do
88 createmany -m $DIR/$tdir/d${i}/f $nfiles > \
89 /dev/null || error "createmany $nfiles"
92 createmany -d $DIR/$tdir/e $ndirs
95 if [ ! -z $igif ]; then
96 touch $DIR/$tdir/dummy
97 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
100 echo "prepared $(date)."
106 #define OBD_FAIL_LFSCK_DELAY1 0x1600
107 do_facet $SINGLEMDS $LCTL set_param fail_val=3 fail_loc=0x1600
108 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
110 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
112 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
113 [ "$STATUS" == "scanning-phase1" ] ||
114 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
116 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
118 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
119 [ "$STATUS" == "stopped" ] ||
120 error "(6) Expect 'stopped', but got '$STATUS'"
122 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
124 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
125 [ "$STATUS" == "scanning-phase1" ] ||
126 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
128 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
129 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
130 mdd.${MDT_DEV}.lfsck_namespace |
131 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
133 error "(9) unexpected status"
136 local repaired=$($SHOW_NAMESPACE |
137 awk '/^updated_phase1/ { print $2 }')
138 [ $repaired -eq 0 ] ||
139 error "(10) Expect nothing to be repaired, but got: $repaired"
141 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
142 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
143 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
144 mdd.${MDT_DEV}.lfsck_namespace |
145 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
147 error "(12) unexpected status"
150 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
151 [ $((scanned1 + 1)) -eq $scanned2 ] ||
152 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
154 echo "stopall, should NOT crash LU-3649"
155 stopall || error "(14) Fail to stopall"
157 run_test 0 "Control LFSCK manually"
160 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
161 skip "OI Scrub not implemented for ZFS" && return
165 #define OBD_FAIL_FID_INDIR 0x1501
166 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
167 touch $DIR/$tdir/dummy
169 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
171 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
172 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
173 mdd.${MDT_DEV}.lfsck_namespace |
174 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
176 error "(4) unexpected status"
179 local repaired=$($SHOW_NAMESPACE |
180 awk '/^updated_phase1/ { print $2 }')
181 [ $repaired -eq 1 ] ||
182 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
184 mount_client $MOUNT || error "(6) Fail to start client!"
186 #define OBD_FAIL_FID_LOOKUP 0x1505
187 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
188 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
190 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
192 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
196 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
197 skip "OI Scrub not implemented for ZFS" && return
201 #define OBD_FAIL_FID_INLMA 0x1502
202 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
203 touch $DIR/$tdir/dummy
205 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
207 #define OBD_FAIL_FID_NOLMA 0x1506
208 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
209 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
210 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
211 mdd.${MDT_DEV}.lfsck_namespace |
212 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
214 error "(4) unexpected status"
217 local repaired=$($SHOW_NAMESPACE |
218 awk '/^updated_phase1/ { print $2 }')
219 [ $repaired -eq 1 ] ||
220 error "(5) Fail to repair missed FID-in-LMA: $repaired"
222 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
223 mount_client $MOUNT || error "(6) Fail to start client!"
225 #define OBD_FAIL_FID_LOOKUP 0x1505
226 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
227 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
229 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
231 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
236 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
237 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
238 touch $DIR/$tdir/dummy
240 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
242 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
243 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
244 mdd.${MDT_DEV}.lfsck_namespace |
245 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
247 error "(4) unexpected status"
250 local repaired=$($SHOW_NAMESPACE |
251 awk '/^updated_phase1/ { print $2 }')
252 [ $repaired -eq 1 ] ||
253 error "(5) Fail to repair crashed linkEA: $repaired"
255 mount_client $MOUNT || error "(6) Fail to start client!"
257 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
258 error "(7) Fail to stat $DIR/$tdir/dummy"
260 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
261 local dummyname=$($LFS fid2path $DIR $dummyfid)
262 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
263 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
265 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
271 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
272 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
273 touch $DIR/$tdir/dummy
275 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
277 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
278 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
279 mdd.${MDT_DEV}.lfsck_namespace |
280 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
282 error "(4) unexpected status"
285 local repaired=$($SHOW_NAMESPACE |
286 awk '/^updated_phase2/ { print $2 }')
287 [ $repaired -eq 1 ] ||
288 error "(5) Fail to repair crashed linkEA: $repaired"
290 mount_client $MOUNT || error "(6) Fail to start client!"
292 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
293 error "(7) Fail to stat $DIR/$tdir/dummy"
295 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
296 local dummyname=$($LFS fid2path $DIR $dummyfid)
297 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
298 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
300 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
306 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
307 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
308 touch $DIR/$tdir/dummy
310 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
312 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
313 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
314 mdd.${MDT_DEV}.lfsck_namespace |
315 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
317 error "(4) unexpected status"
320 local repaired=$($SHOW_NAMESPACE |
321 awk '/^updated_phase2/ { print $2 }')
322 [ $repaired -eq 1 ] ||
323 error "(5) Fail to repair crashed linkEA: $repaired"
325 mount_client $MOUNT || error "(6) Fail to start client!"
327 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
328 error "(7) Fail to stat $DIR/$tdir/dummy"
330 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
331 local dummyname=$($LFS fid2path $DIR $dummyfid)
332 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
333 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
335 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
339 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
340 skip "OI Scrub not implemented for ZFS" && return
343 cleanup_mount $MOUNT || error "(0.1) Fail to stop client!"
344 stop $SINGLEMDS > /dev/null || error "(0.2) Fail to stop MDS!"
346 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
347 echo "start $SINGLEMDS with disabling OI scrub"
348 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
349 error "(2) Fail to start MDS!"
351 #define OBD_FAIL_LFSCK_DELAY2 0x1601
352 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
353 $START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
354 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
355 mdd.${MDT_DEV}.lfsck_namespace |
356 awk '/^flags/ { print \\\$2 }'" "inconsistent" 6 || {
358 error "(5) unexpected status"
361 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
362 [ "$STATUS" == "scanning-phase1" ] ||
363 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
365 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
366 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
367 mdd.${MDT_DEV}.lfsck_namespace |
368 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
370 error "(7) unexpected status"
373 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
374 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
376 local repaired=$($SHOW_NAMESPACE |
377 awk '/^updated_phase1/ { print $2 }')
378 [ $repaired -ge 9 ] ||
379 error "(9) Fail to repair crashed linkEA: $repaired"
381 mount_client $MOUNT || error "(10) Fail to start client!"
383 #define OBD_FAIL_FID_LOOKUP 0x1505
384 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
385 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
386 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
388 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
392 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
393 skip "OI Scrub not implemented for ZFS" && return
396 cleanup_mount $MOUNT || error "(0.1) Fail to stop client!"
397 stop $SINGLEMDS > /dev/null || error "(0.2) Fail to stop MDS!"
399 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
400 echo "start $SINGLEMDS with disabling OI scrub"
401 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
402 error "(2) Fail to start MDS!"
404 #define OBD_FAIL_LFSCK_DELAY2 0x1601
405 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
406 $START_NAMESPACE -r || error "(4) Fail to start LFSCK for namespace!"
407 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
408 mdd.${MDT_DEV}.lfsck_namespace |
409 awk '/^flags/ { print \\\$2 }'" "inconsistent,upgrade" 6 || {
411 error "(5) unexpected status"
414 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
415 [ "$STATUS" == "scanning-phase1" ] ||
416 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
418 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
419 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
420 mdd.${MDT_DEV}.lfsck_namespace |
421 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
423 error "(7) unexpected status"
426 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
427 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
429 local repaired=$($SHOW_NAMESPACE |
430 awk '/^updated_phase1/ { print $2 }')
431 [ $repaired -ge 2 ] ||
432 error "(9) Fail to repair crashed linkEA: $repaired"
434 mount_client $MOUNT || error "(10) Fail to start client!"
436 #define OBD_FAIL_FID_LOOKUP 0x1505
437 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
438 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
440 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
442 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
443 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
444 local dummyname=$($LFS fid2path $DIR $dummyfid)
445 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
446 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
448 run_test 5 "LFSCK can handle IGIF object upgrading"
453 #define OBD_FAIL_LFSCK_DELAY1 0x1600
454 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1600
455 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
457 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
458 [ "$STATUS" == "scanning-phase1" ] ||
459 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
461 # Sleep 3 sec to guarantee at least one object processed by LFSCK
463 # Fail the LFSCK to guarantee there is at least one checkpoint
464 #define OBD_FAIL_LFSCK_FATAL1 0x1608
465 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
466 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
467 mdd.${MDT_DEV}.lfsck_namespace |
468 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
470 error "(4) unexpected status"
473 local POS0=$($SHOW_NAMESPACE |
474 awk '/^last_checkpoint_position/ { print $2 }' |
477 #define OBD_FAIL_LFSCK_DELAY1 0x1600
478 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1600
479 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
481 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
482 [ "$STATUS" == "scanning-phase1" ] ||
483 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
485 local POS1=$($SHOW_NAMESPACE |
486 awk '/^latest_start_position/ { print $2 }' |
488 [ $POS0 -lt $POS1 ] ||
489 error "(7) Expect larger than: $POS0, but got $POS1"
491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
492 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
493 mdd.${MDT_DEV}.lfsck_namespace |
494 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
496 error "(8) unexpected status"
499 run_test 6a "LFSCK resumes from last checkpoint (1)"
504 #define OBD_FAIL_LFSCK_DELAY2 0x1601
505 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
506 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
508 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
509 [ "$STATUS" == "scanning-phase1" ] ||
510 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
512 # Sleep 5 sec to guarantee that we are in the directory scanning
514 # Fail the LFSCK to guarantee there is at least one checkpoint
515 #define OBD_FAIL_LFSCK_FATAL2 0x1609
516 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
517 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
518 mdd.${MDT_DEV}.lfsck_namespace |
519 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
521 error "(4) unexpected status"
524 local O_POS0=$($SHOW_NAMESPACE |
525 awk '/^last_checkpoint_position/ { print $2 }' |
528 local D_POS0=$($SHOW_NAMESPACE |
529 awk '/^last_checkpoint_position/ { print $4 }')
531 #define OBD_FAIL_LFSCK_DELAY2 0x1601
532 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
533 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
535 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
536 [ "$STATUS" == "scanning-phase1" ] ||
537 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
539 local O_POS1=$($SHOW_NAMESPACE |
540 awk '/^latest_start_position/ { print $2 }' |
542 local D_POS1=$($SHOW_NAMESPACE |
543 awk '/^latest_start_position/ { print $4 }')
545 if [ "$D_POS0" == "N/A" -o "$D_POS1" == "N/A" ]; then
546 [ $O_POS0 -lt $O_POS1 ] ||
547 error "(7.1) $O_POS1 is not larger than $O_POS0"
549 [ $D_POS0 -lt $D_POS1 ] ||
550 error "(7.2) $D_POS1 is not larger than $D_POS0"
553 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
554 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
555 mdd.${MDT_DEV}.lfsck_namespace |
556 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
558 error "(8) unexpected status"
561 run_test 6b "LFSCK resumes from last checkpoint (2)"
568 #define OBD_FAIL_LFSCK_DELAY2 0x1601
569 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1601
570 $START_NAMESPACE -r || error "(2) Fail to start LFSCK for namespace!"
572 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "scanning-phase1" ] ||
574 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
576 # Sleep 3 sec to guarantee at least one object processed by LFSCK
578 echo "stop $SINGLEMDS"
579 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(5) Fail to start MDS!"
585 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
586 [ "$STATUS" == "scanning-phase1" ] ||
587 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
589 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
590 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
591 mdd.${MDT_DEV}.lfsck_namespace |
592 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
594 error "(7) unexpected status"
597 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
603 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
604 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
605 for ((i = 0; i < 20; i++)); do
606 touch $DIR/$tdir/dummy${i}
609 #define OBD_FAIL_LFSCK_DELAY3 0x1602
610 do_facet $SINGLEMDS $LCTL set_param fail_val=1 fail_loc=0x1602
611 $START_NAMESPACE -r || error "(3) Fail to start LFSCK for namespace!"
612 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
613 mdd.${MDT_DEV}.lfsck_namespace |
614 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
616 error "(4) unexpected status"
619 echo "stop $SINGLEMDS"
620 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
622 echo "start $SINGLEMDS"
623 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
624 error "(6) Fail to start MDS!"
626 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
627 [ "$STATUS" == "scanning-phase2" ] ||
628 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
630 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
631 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
632 mdd.${MDT_DEV}.lfsck_namespace |
633 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
635 error "(8) unexpected status"
638 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
643 formatall > /dev/null
649 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
650 [ "$STATUS" == "init" ] ||
651 error "(2) Expect 'init', but got '$STATUS'"
653 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
654 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
655 mkdir $DIR/$tdir/crashed
657 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
658 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
659 for ((i = 0; i < 5; i++)); do
660 touch $DIR/$tdir/dummy${i}
663 umount_client $MOUNT || error "(3) Fail to stop client!"
665 #define OBD_FAIL_LFSCK_DELAY2 0x1601
666 do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1601
667 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
669 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
670 [ "$STATUS" == "scanning-phase1" ] ||
671 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
673 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
675 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
676 [ "$STATUS" == "stopped" ] ||
677 error "(7) Expect 'stopped', but got '$STATUS'"
679 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
681 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
682 [ "$STATUS" == "scanning-phase1" ] ||
683 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
685 #define OBD_FAIL_LFSCK_FATAL2 0x1609
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
687 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
688 mdd.${MDT_DEV}.lfsck_namespace |
689 awk '/^status/ { print \\\$2 }'" "failed" 6 || {
691 error "(10) unexpected status"
694 #define OBD_FAIL_LFSCK_DELAY1 0x1600
695 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
696 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
698 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
699 [ "$STATUS" == "scanning-phase1" ] ||
700 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
702 #define OBD_FAIL_LFSCK_CRASH 0x160a
703 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
706 echo "stop $SINGLEMDS"
707 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
709 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
710 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
712 echo "start $SINGLEMDS"
713 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
714 error "(14) Fail to start MDS!"
716 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "crashed" ] ||
718 error "(15) Expect 'crashed', but got '$STATUS'"
720 #define OBD_FAIL_LFSCK_DELAY2 0x1601
721 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
722 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
724 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
725 [ "$STATUS" == "scanning-phase1" ] ||
726 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
728 echo "stop $SINGLEMDS"
729 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
731 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
732 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
734 echo "start $SINGLEMDS"
735 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
736 error "(19) Fail to start MDS!"
738 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
739 [ "$STATUS" == "paused" ] ||
740 error "(20) Expect 'paused', but got '$STATUS'"
742 #define OBD_FAIL_LFSCK_DELAY3 0x1602
743 do_facet $SINGLEMDS $LCTL set_param fail_val=2 fail_loc=0x1602
745 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
746 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
747 mdd.${MDT_DEV}.lfsck_namespace |
748 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 || {
750 error "(22) unexpected status"
753 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
754 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
755 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
757 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
758 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
759 mdd.${MDT_DEV}.lfsck_namespace |
760 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
762 error "(24) unexpected status"
765 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
766 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
768 run_test 8 "LFSCK state machine"
771 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
772 skip "Testing on UP system, the speed may be inaccurate."
778 local BASE_SPEED1=100
780 $START_NAMESPACE -r -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
783 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
784 [ "$STATUS" == "scanning-phase1" ] ||
785 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
787 local SPEED=$($SHOW_NAMESPACE |
788 awk '/^average_speed_phase1/ { print $2 }')
790 # There may be time error, normally it should be less than 2 seconds.
791 # We allow another 20% schedule error.
793 # MAX_MARGIN = 1.2 = 12 / 10
794 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
795 RUN_TIME1 * 12 / 10))
796 [ $SPEED -lt $MAX_SPEED ] ||
797 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
800 local BASE_SPEED2=300
802 do_facet $SINGLEMDS \
803 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
806 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
807 # MIN_MARGIN = 0.8 = 8 / 10
808 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
809 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
810 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
811 [ $SPEED -gt $MIN_SPEED ] ||
812 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
814 # MAX_MARGIN = 1.2 = 12 / 10
815 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
816 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
817 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
818 [ $SPEED -lt $MAX_SPEED ] ||
819 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
821 do_facet $SINGLEMDS \
822 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
824 wait_update_facet $SINGLEMDS \
825 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
826 awk '/^status/ { print \\\$2 }'" "completed" 30 ||
827 error "(7) Failed to get expected 'completed'"
829 run_test 9a "LFSCK speed control (1)"
832 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
833 skip "Testing on UP system, the speed may be inaccurate."
839 echo "Preparing another 50 * 50 files (with error) at $(date)."
840 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
841 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
842 createmany -d $DIR/$tdir/d 50
843 createmany -m $DIR/$tdir/f 50
844 for ((i = 0; i < 50; i++)); do
845 createmany -m $DIR/$tdir/d${i}/f 50 > /dev/null
848 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
849 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
850 $START_NAMESPACE -r || error "(4) Fail to start LFSCK!"
851 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
852 mdd.${MDT_DEV}.lfsck_namespace |
853 awk '/^status/ { print \\\$2 }'" "stopped" 10 || {
855 error "(5) unexpected status"
858 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
859 echo "Prepared at $(date)."
863 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
866 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
867 [ "$STATUS" == "scanning-phase2" ] ||
868 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
870 local SPEED=$($SHOW_NAMESPACE |
871 awk '/^average_speed_phase2/ { print $2 }')
872 # There may be time error, normally it should be less than 2 seconds.
873 # We allow another 20% schedule error.
875 # MAX_MARGIN = 1.2 = 12 / 10
876 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
877 RUN_TIME1 * 12 / 10))
878 [ $SPEED -lt $MAX_SPEED ] ||
879 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
882 local BASE_SPEED2=150
884 do_facet $SINGLEMDS \
885 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
888 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
889 # MIN_MARGIN = 0.8 = 8 / 10
890 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
891 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
892 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
893 [ $SPEED -gt $MIN_SPEED ] ||
894 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
896 # MAX_MARGIN = 1.2 = 12 / 10
897 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
898 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
899 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
900 [ $SPEED -lt $MAX_SPEED ] ||
901 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
903 do_facet $SINGLEMDS \
904 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
905 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
906 mdd.${MDT_DEV}.lfsck_namespace |
907 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
909 error "(11) unexpected status"
912 run_test 9b "LFSCK speed control (2)"
916 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
917 skip "lookup(..)/linkea on ZFS issue" && return
921 echo "Preparing more files with error at $(date)."
922 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
923 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
925 for ((i = 0; i < 1000; i = $((i+2)))); do
926 mkdir -p $DIR/$tdir/d${i}
927 touch $DIR/$tdir/f${i}
928 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
931 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
932 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
934 for ((i = 1; i < 1000; i = $((i+2)))); do
935 mkdir -p $DIR/$tdir/d${i}
936 touch $DIR/$tdir/f${i}
937 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
940 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
941 echo "Prepared at $(date)."
943 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
946 mount_client $MOUNT || error "(3) Fail to start client!"
948 $START_NAMESPACE -r -s 100 || error "(5) Fail to start LFSCK!"
951 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
952 [ "$STATUS" == "scanning-phase1" ] ||
953 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
955 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
957 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
959 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
961 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
963 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
965 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
967 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
969 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
970 error "(14) Fail to softlink!"
972 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
973 [ "$STATUS" == "scanning-phase1" ] ||
974 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
976 do_facet $SINGLEMDS \
977 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
978 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
979 mdd.${MDT_DEV}.lfsck_namespace |
980 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
982 error "(16) unexpected status"
985 run_test 10 "System is available during LFSCK scanning"
988 ost_remove_lastid() {
991 local rcmd="do_facet ost${ost}"
993 echo "remove LAST_ID on ost${ost}: idx=${idx}"
995 # step 1: local mount
996 mount_fstype ost${ost} || return 1
997 # step 2: remove the specified LAST_ID
998 ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/{LAST_ID,d0/0}
1000 unmount_fstype ost${ost} || return 2
1004 check_mount_and_prep
1005 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1006 createmany -o $DIR/$tdir/f 64 || error "(0) Fail to create 64 files."
1011 ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
1013 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
1014 error "(2) Fail to start ost1"
1016 #define OBD_FAIL_LFSCK_DELAY4 0x160e
1017 do_facet ost1 $LCTL set_param fail_val=3 fail_loc=0x160e
1019 echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
1020 $START_LAYOUT_ON_OST -r || error "(4) Fail to start LFSCK on OST!"
1022 wait_update_facet ost1 "$LCTL get_param -n \
1023 obdfilter.${OST_DEV}.lfsck_layout |
1024 awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
1026 error "(5) unexpected status"
1029 do_facet ost1 $LCTL set_param fail_val=0 fail_loc=0
1031 wait_update_facet ost1 "$LCTL get_param -n \
1032 obdfilter.${OST_DEV}.lfsck_layout |
1033 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1035 error "(6) unexpected status"
1038 echo "the LAST_ID(s) should have been rebuilt"
1039 FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
1040 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
1042 run_test 11a "LFSCK can rebuild lost last_id"
1045 check_mount_and_prep
1046 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1048 echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
1049 #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
1050 do_facet ost1 $LCTL set_param fail_loc=0x160d
1051 createmany -o $DIR/$tdir/f 64
1052 local lastid1=$(do_facet ost1 "lctl get_param -n \
1053 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1054 awk -F: '{ print $2 }')
1056 umount_client $MOUNT
1057 stop ost1 || error "(1) Fail to stop ost1"
1059 #define OBD_FAIL_OST_ENOSPC 0x215
1060 do_facet ost1 $LCTL set_param fail_loc=0x215
1062 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1063 error "(2) Fail to start ost1"
1065 for ((i = 0; i < 60; i++)); do
1066 lastid2=$(do_facet ost1 "lctl get_param -n \
1067 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1068 awk -F: '{ print $2 }')
1069 [ ! -z $lastid2 ] && break;
1073 echo "the on-disk LAST_ID should be smaller than the expected one"
1074 [ $lastid1 -gt $lastid2 ] ||
1075 error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
1077 echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
1078 $START_LAYOUT_ON_OST -r || error "(5) Fail to start LFSCK on OST!"
1080 wait_update_facet ost1 "$LCTL get_param -n \
1081 obdfilter.${OST_DEV}.lfsck_layout |
1082 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1084 error "(6) unexpected status"
1087 stop ost1 || error "(7) Fail to stop ost1"
1089 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1090 error "(8) Fail to start ost1"
1092 echo "the on-disk LAST_ID should have been rebuilt"
1093 wait_update_facet ost1 "$LCTL get_param -n \
1094 obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
1095 awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
1096 $LCTL get_param -n obdfilter.${ost1_svc}.last_id
1097 error "(9) expect lastid1 0x100000000:$lastid1"
1100 do_facet ost1 $LCTL set_param fail_loc=0
1101 stopall || error "(10) Fail to stopall"
1103 run_test 11b "LFSCK can rebuild crashed last_id"
1106 [ $MDSCOUNT -lt 2 ] &&
1107 skip "We need at least 2 MDSes for test_12" && exit 0
1109 check_mount_and_prep
1110 for k in $(seq $MDSCOUNT); do
1111 $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
1112 createmany -o $DIR/$tdir/${k}/f 100 ||
1113 error "(0) Fail to create 100 files."
1116 echo "Start namespace LFSCK on all targets by single command (-s 1)."
1117 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1118 -s 1 -r || error "(2) Fail to start LFSCK on all devices!"
1120 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1121 for k in $(seq $MDSCOUNT); do
1122 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1123 mdd.$(facet_svc mds${k}).lfsck_namespace |
1124 awk '/^status/ { print $2 }')
1125 [ "$STATUS" == "scanning-phase1" ] ||
1126 error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1129 echo "Stop namespace LFSCK on all targets by single lctl command."
1130 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1131 error "(4) Fail to stop LFSCK on all devices!"
1133 echo "All the LFSCK targets should be in 'stopped' status."
1134 for k in $(seq $MDSCOUNT); do
1135 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1136 mdd.$(facet_svc mds${k}).lfsck_namespace |
1137 awk '/^status/ { print $2 }')
1138 [ "$STATUS" == "stopped" ] ||
1139 error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
1142 echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
1143 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1144 -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
1146 echo "All the LFSCK targets should be in 'completed' status."
1147 for k in $(seq $MDSCOUNT); do
1148 wait_update_facet mds${k} "$LCTL get_param -n \
1149 mdd.$(facet_svc mds${k}).lfsck_namespace |
1150 awk '/^status/ { print \\\$2 }'" "completed" 8 ||
1151 error "(7) MDS${k} is not the expected 'completed'"
1154 echo "Start layout LFSCK on all targets by single command (-s 1)."
1155 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1156 -s 1 -r || error "(8) Fail to start LFSCK on all devices!"
1158 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1159 for k in $(seq $MDSCOUNT); do
1160 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1161 mdd.$(facet_svc mds${k}).lfsck_layout |
1162 awk '/^status/ { print $2 }')
1163 [ "$STATUS" == "scanning-phase1" ] ||
1164 error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1167 echo "Stop layout LFSCK on all targets by single lctl command."
1168 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1169 error "(10) Fail to stop LFSCK on all devices!"
1171 echo "All the LFSCK targets should be in 'stopped' status."
1172 for k in $(seq $MDSCOUNT); do
1173 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1174 mdd.$(facet_svc mds${k}).lfsck_layout |
1175 awk '/^status/ { print $2 }')
1176 [ "$STATUS" == "stopped" ] ||
1177 error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
1180 for k in $(seq $OSTCOUNT); do
1181 local STATUS=$(do_facet ost${k} $LCTL get_param -n \
1182 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1183 awk '/^status/ { print $2 }')
1184 [ "$STATUS" == "stopped" ] ||
1185 error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
1188 echo "Re-start layout LFSCK on all targets by single command (-s 0)."
1189 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1190 -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
1192 echo "All the LFSCK targets should be in 'completed' status."
1193 for k in $(seq $MDSCOUNT); do
1194 # The LFSCK status query internal is 30 seconds. For the case
1195 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1196 # time to guarantee the status sync up.
1197 wait_update_facet mds${k} "$LCTL get_param -n \
1198 mdd.$(facet_svc mds${k}).lfsck_layout |
1199 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1200 error "(14) MDS${k} is not the expected 'completed'"
1203 run_test 12 "single command to trigger LFSCK on all devices"
1207 echo "The lmm_oi in layout EA should be consistent with the MDT-object"
1208 echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
1209 echo "MDT-object FID."
1212 check_mount_and_prep
1214 echo "Inject failure stub to simulate bad lmm_oi"
1215 #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
1216 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
1217 createmany -o $DIR/$tdir/f 32
1218 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1220 echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
1221 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1223 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1224 mdd.${MDT_DEV}.lfsck_layout |
1225 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1227 error "(2) unexpected status"
1230 local repaired=$($SHOW_LAYOUT |
1231 awk '/^repaired_others/ { print $2 }')
1232 [ $repaired -eq 32 ] ||
1233 error "(3) Fail to repair crashed lmm_oi: $repaired"
1235 run_test 13 "LFSCK can repair crashed lmm_oi"
1239 echo "The OST-object referenced by the MDT-object should be there;"
1240 echo "otherwise, the LFSCK should re-create the missed OST-object."
1243 check_mount_and_prep
1244 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1246 local count=$(precreated_ost_obj_count 0 0)
1248 echo "Inject failure stub to simulate dangling referenced MDT-object"
1249 #define OBD_FAIL_LFSCK_DANGLING 0x1610
1250 do_facet ost1 $LCTL set_param fail_loc=0x1610
1251 createmany -o $DIR/$tdir/f $((count + 32))
1252 do_facet ost1 $LCTL set_param fail_loc=0
1254 # exhaust other pre-created dangling cases
1255 count=$(precreated_ost_obj_count 0 0)
1256 createmany -o $DIR/$tdir/a $count ||
1257 error "(0) Fail to create $count files."
1259 echo "'ls' should fail because of dangling referenced MDT-object"
1260 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
1262 echo "Trigger layout LFSCK to find out dangling reference"
1263 $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
1265 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1266 mdd.${MDT_DEV}.lfsck_layout |
1267 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1269 error "(3) unexpected status"
1272 local repaired=$($SHOW_LAYOUT |
1273 awk '/^repaired_dangling/ { print $2 }')
1274 [ $repaired -ge 32 ] ||
1275 error "(4) Fail to repair dangling reference: $repaired"
1277 echo "'ls' should fail because it will not repair dangling by default"
1278 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail."
1280 echo "Trigger layout LFSCK to repair dangling reference"
1281 $START_LAYOUT -r -c || error "(6) Fail to start LFSCK for layout!"
1283 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1284 mdd.${MDT_DEV}.lfsck_layout |
1285 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1287 error "(7) unexpected status"
1290 repaired=$($SHOW_LAYOUT |
1291 awk '/^repaired_dangling/ { print $2 }')
1292 [ $repaired -ge 32 ] ||
1293 error "(8) Fail to repair dangling reference: $repaired"
1295 echo "'ls' should success after layout LFSCK repairing"
1296 ls -ail $DIR/$tdir > /dev/null || error "(9) ls should success."
1298 run_test 14 "LFSCK can repair MDT-object with dangling reference"
1302 echo "If the OST-object referenced by the MDT-object back points"
1303 echo "to some non-exist MDT-object, then the LFSCK should repair"
1304 echo "the OST-object to back point to the right MDT-object."
1307 check_mount_and_prep
1308 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1310 echo "Inject failure stub to make the OST-object to back point to"
1311 echo "non-exist MDT-object."
1312 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
1314 do_facet ost1 $LCTL set_param fail_loc=0x1611
1315 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1316 cancel_lru_locks osc
1317 do_facet ost1 $LCTL set_param fail_loc=0
1319 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1320 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1322 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1323 mdd.${MDT_DEV}.lfsck_layout |
1324 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1326 error "(2) unexpected status"
1329 local repaired=$($SHOW_LAYOUT |
1330 awk '/^repaired_unmatched_pair/ { print $2 }')
1331 [ $repaired -eq 1 ] ||
1332 error "(3) Fail to repair unmatched pair: $repaired"
1334 run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
1338 echo "If the OST-object referenced by the MDT-object back points"
1339 echo "to other MDT-object that doesn't recognize the OST-object,"
1340 echo "then the LFSCK should repair it to back point to the right"
1341 echo "MDT-object (the first one)."
1344 check_mount_and_prep
1345 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1346 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1347 cancel_lru_locks osc
1349 echo "Inject failure stub to make the OST-object to back point to"
1350 echo "other MDT-object"
1352 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
1353 do_facet ost1 $LCTL set_param fail_loc=0x1612
1354 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1355 cancel_lru_locks osc
1356 do_facet ost1 $LCTL set_param fail_loc=0
1358 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1359 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1361 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1362 mdd.${MDT_DEV}.lfsck_layout |
1363 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1365 error "(2) unexpected status"
1368 local repaired=$($SHOW_LAYOUT |
1369 awk '/^repaired_unmatched_pair/ { print $2 }')
1370 [ $repaired -eq 1 ] ||
1371 error "(3) Fail to repair unmatched pair: $repaired"
1373 run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
1377 echo "If the OST-object's owner information does not match the owner"
1378 echo "information stored in the MDT-object, then the LFSCK trust the"
1379 echo "MDT-object and update the OST-object's owner information."
1382 check_mount_and_prep
1383 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1384 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1385 cancel_lru_locks osc
1387 echo "Inject failure stub to skip OST-object owner changing"
1388 #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
1389 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
1390 chown 1.1 $DIR/$tdir/f0
1391 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1393 echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
1396 $START_LAYOUT -r || error "(1) Fail to start LFSCK for layout!"
1398 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1399 mdd.${MDT_DEV}.lfsck_layout |
1400 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1402 error "(2) unexpected status"
1405 local repaired=$($SHOW_LAYOUT |
1406 awk '/^repaired_inconsistent_owner/ { print $2 }')
1407 [ $repaired -eq 1 ] ||
1408 error "(3) Fail to repair inconsistent owner: $repaired"
1410 run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
1414 echo "If more than one MDT-objects reference the same OST-object,"
1415 echo "and the OST-object only recognizes one MDT-object, then the"
1416 echo "LFSCK should create new OST-objects for such non-recognized"
1420 check_mount_and_prep
1421 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1423 echo "Inject failure stub to make two MDT-objects to refernce"
1424 echo "the OST-object"
1426 #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
1427 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0x1614
1429 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1430 cancel_lru_locks osc
1432 createmany -o $DIR/$tdir/f 1
1434 do_facet $SINGLEMDS $LCTL set_param fail_loc=0 fail_val=0
1436 cancel_lru_locks mdc
1437 cancel_lru_locks osc
1439 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
1440 local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
1441 [ $size -eq 1048576 ] ||
1442 error "(1) f0 (wrong) size should be 1048576, but got $size"
1444 echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
1447 $START_LAYOUT -r || error "(2) Fail to start LFSCK for layout!"
1449 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1450 mdd.${MDT_DEV}.lfsck_layout |
1451 awk '/^status/ { print \\\$2 }'" "completed" 6 || {
1453 error "(3) unexpected status"
1456 local repaired=$($SHOW_LAYOUT |
1457 awk '/^repaired_multiple_referenced/ { print $2 }')
1458 [ $repaired -eq 1 ] ||
1459 error "(4) Fail to repair multiple references: $repaired"
1461 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
1462 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
1463 error "(5) Fail to write f0."
1464 size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
1465 [ $size -eq 1048576 ] ||
1466 error "(6) guard size should be 1048576, but got $size"
1468 run_test 17 "LFSCK can repair multiple references"
1472 echo "The target MDT-object is there, but related stripe information"
1473 echo "is lost or partly lost. The LFSCK should regenerate the missed"
1474 echo "layout EA entries."
1477 check_mount_and_prep
1478 $LFS mkdir -i 0 $DIR/$tdir/a1
1479 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1480 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1482 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1484 $LFS path2fid $DIR/$tdir/a1/f1
1485 $LFS getstripe $DIR/$tdir/a1/f1
1487 if [ $MDSCOUNT -ge 2 ]; then
1488 $LFS mkdir -i 1 $DIR/$tdir/a2
1489 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1490 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1491 $LFS path2fid $DIR/$tdir/a2/f2
1492 $LFS getstripe $DIR/$tdir/a2/f2
1495 cancel_lru_locks osc
1497 echo "Inject failure, to make the MDT-object lost its layout EA"
1498 #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
1499 do_facet mds1 $LCTL set_param fail_loc=0x1615
1500 chown 1.1 $DIR/$tdir/a1/f1
1502 if [ $MDSCOUNT -ge 2 ]; then
1503 do_facet mds2 $LCTL set_param fail_loc=0x1615
1504 chown 1.1 $DIR/$tdir/a2/f2
1510 do_facet mds1 $LCTL set_param fail_loc=0
1511 if [ $MDSCOUNT -ge 2 ]; then
1512 do_facet mds2 $LCTL set_param fail_loc=0
1515 cancel_lru_locks mdc
1516 cancel_lru_locks osc
1518 echo "The file size should be incorrect since layout EA is lost"
1519 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1520 [ "$cur_size" != "$saved_size" ] ||
1521 error "(1) Expect incorrect file1 size"
1523 if [ $MDSCOUNT -ge 2 ]; then
1524 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1525 [ "$cur_size" != "$saved_size" ] ||
1526 error "(2) Expect incorrect file2 size"
1529 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1530 $START_LAYOUT -r -o || error "(3) Fail to start LFSCK for layout!"
1532 for k in $(seq $MDSCOUNT); do
1533 # The LFSCK status query internal is 30 seconds. For the case
1534 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1535 # time to guarantee the status sync up.
1536 wait_update_facet mds${k} "$LCTL get_param -n \
1537 mdd.$(facet_svc mds${k}).lfsck_layout |
1538 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1539 error "(4) MDS${k} is not the expected 'completed'"
1542 for k in $(seq $OSTCOUNT); do
1543 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1544 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1545 awk '/^status/ { print $2 }')
1546 [ "$cur_status" == "completed" ] ||
1547 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1550 local repaired=$(do_facet mds1 $LCTL get_param -n \
1551 mdd.$(facet_svc mds1).lfsck_layout |
1552 awk '/^repaired_orphan/ { print $2 }')
1553 [ $repaired -eq 1 ] ||
1554 error "(6.1) Expect 1 fixed on mds1, but got: $repaired"
1556 if [ $MDSCOUNT -ge 2 ]; then
1557 repaired=$(do_facet mds2 $LCTL get_param -n \
1558 mdd.$(facet_svc mds2).lfsck_layout |
1559 awk '/^repaired_orphan/ { print $2 }')
1560 [ $repaired -eq 2 ] ||
1561 error "(6.2) Expect 2 fixed on mds2, but got: $repaired"
1564 $LFS path2fid $DIR/$tdir/a1/f1
1565 $LFS getstripe $DIR/$tdir/a1/f1
1567 if [ $MDSCOUNT -ge 2 ]; then
1568 $LFS path2fid $DIR/$tdir/a2/f2
1569 $LFS getstripe $DIR/$tdir/a2/f2
1572 echo "The file size should be correct after layout LFSCK scanning"
1573 cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1574 [ "$cur_size" == "$saved_size" ] ||
1575 error "(7) Expect file1 size $saved_size, but got $cur_size"
1577 if [ $MDSCOUNT -ge 2 ]; then
1578 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1579 [ "$cur_size" == "$saved_size" ] ||
1580 error "(8) Expect file2 size $saved_size, but got $cur_size"
1583 run_test 18a "Find out orphan OST-object and repair it (1)"
1587 echo "The target MDT-object is lost. The LFSCK should re-create the"
1588 echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
1589 echo "can move it back to normal namespace manually."
1592 check_mount_and_prep
1593 $LFS mkdir -i 0 $DIR/$tdir/a1
1594 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1595 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1596 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1597 local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
1599 $LFS getstripe $DIR/$tdir/a1/f1
1601 if [ $MDSCOUNT -ge 2 ]; then
1602 $LFS mkdir -i 1 $DIR/$tdir/a2
1603 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1604 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1605 fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
1607 $LFS getstripe $DIR/$tdir/a2/f2
1610 cancel_lru_locks osc
1612 echo "Inject failure, to simulate the case of missing the MDT-object"
1613 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1614 do_facet mds1 $LCTL set_param fail_loc=0x1616
1615 rm -f $DIR/$tdir/a1/f1
1617 if [ $MDSCOUNT -ge 2 ]; then
1618 do_facet mds2 $LCTL set_param fail_loc=0x1616
1619 rm -f $DIR/$tdir/a2/f2
1625 do_facet mds1 $LCTL set_param fail_loc=0
1626 if [ $MDSCOUNT -ge 2 ]; then
1627 do_facet mds2 $LCTL set_param fail_loc=0
1630 cancel_lru_locks mdc
1631 cancel_lru_locks osc
1633 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1634 $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
1636 for k in $(seq $MDSCOUNT); do
1637 # The LFSCK status query internal is 30 seconds. For the case
1638 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1639 # time to guarantee the status sync up.
1640 wait_update_facet mds${k} "$LCTL get_param -n \
1641 mdd.$(facet_svc mds${k}).lfsck_layout |
1642 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1643 error "(2) MDS${k} is not the expected 'completed'"
1646 for k in $(seq $OSTCOUNT); do
1647 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1648 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1649 awk '/^status/ { print $2 }')
1650 [ "$cur_status" == "completed" ] ||
1651 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1654 local repaired=$(do_facet mds1 $LCTL get_param -n \
1655 mdd.$(facet_svc mds1).lfsck_layout |
1656 awk '/^repaired_orphan/ { print $2 }')
1657 [ $repaired -eq 1 ] ||
1658 error "(4.1) Expect 1 fixed on mds1, but got: $repaired"
1660 if [ $MDSCOUNT -ge 2 ]; then
1661 repaired=$(do_facet mds2 $LCTL get_param -n \
1662 mdd.$(facet_svc mds2).lfsck_layout |
1663 awk '/^repaired_orphan/ { print $2 }')
1664 [ $repaired -eq 2 ] ||
1665 error "(4.2) Expect 2 fixed on mds2, but got: $repaired"
1668 echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
1669 mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
1670 error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
1672 if [ $MDSCOUNT -ge 2 ]; then
1673 local name=$MOUNT/.lustre/lost+found/MDT0001/R-${fid2}
1674 mv $name $DIR/$tdir/a2/f2 || error "(6) Fail to move $name"
1677 $LFS path2fid $DIR/$tdir/a1/f1
1678 $LFS getstripe $DIR/$tdir/a1/f1
1680 if [ $MDSCOUNT -ge 2 ]; then
1681 $LFS path2fid $DIR/$tdir/a2/f2
1682 $LFS getstripe $DIR/$tdir/a2/f2
1685 echo "The file size should be correct after layout LFSCK scanning"
1686 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1687 [ "$cur_size" == "$saved_size" ] ||
1688 error "(7) Expect file1 size $saved_size, but got $cur_size"
1690 if [ $MDSCOUNT -ge 2 ]; then
1691 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1692 [ "$cur_size" == "$saved_size" ] ||
1693 error "(8) Expect file2 size $saved_size, but got $cur_size"
1696 run_test 18b "Find out orphan OST-object and repair it (2)"
1700 echo "The target MDT-object is lost, and the OST-object FID is missing."
1701 echo "The LFSCK should re-create the MDT-object with new FID under the "
1702 echo "directory .lustre/lost+found/MDTxxxx."
1705 check_mount_and_prep
1706 $LFS mkdir -i 0 $DIR/$tdir/a1
1707 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1709 echo "Inject failure, to simulate the case of missing parent FID"
1710 #define OBD_FAIL_LFSCK_NOPFID 0x1617
1711 do_facet ost1 $LCTL set_param fail_loc=0x1617
1713 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1714 $LFS getstripe $DIR/$tdir/a1/f1
1716 if [ $MDSCOUNT -ge 2 ]; then
1717 $LFS mkdir -i 1 $DIR/$tdir/a2
1718 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1719 do_facet ost2 $LCTL set_param fail_loc=0x1617
1720 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1721 $LFS getstripe $DIR/$tdir/a2/f2
1724 cancel_lru_locks osc
1726 echo "Inject failure, to simulate the case of missing the MDT-object"
1727 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1728 do_facet mds1 $LCTL set_param fail_loc=0x1616
1729 rm -f $DIR/$tdir/a1/f1
1731 if [ $MDSCOUNT -ge 2 ]; then
1732 do_facet mds2 $LCTL set_param fail_loc=0x1616
1733 rm -f $DIR/$tdir/a2/f2
1739 do_facet mds1 $LCTL set_param fail_loc=0
1740 if [ $MDSCOUNT -ge 2 ]; then
1741 do_facet mds2 $LCTL set_param fail_loc=0
1744 cancel_lru_locks mdc
1745 cancel_lru_locks osc
1747 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1748 $START_LAYOUT -r -o || error "(1) Fail to start LFSCK for layout!"
1750 for k in $(seq $MDSCOUNT); do
1751 # The LFSCK status query internal is 30 seconds. For the case
1752 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1753 # time to guarantee the status sync up.
1754 wait_update_facet mds${k} "$LCTL get_param -n \
1755 mdd.$(facet_svc mds${k}).lfsck_layout |
1756 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1757 error "(2) MDS${k} is not the expected 'completed'"
1760 for k in $(seq $OSTCOUNT); do
1761 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1762 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1763 awk '/^status/ { print $2 }')
1764 [ "$cur_status" == "completed" ] ||
1765 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1768 if [ $MDSCOUNT -ge 2 ]; then
1774 local repaired=$(do_facet mds1 $LCTL get_param -n \
1775 mdd.$(facet_svc mds1).lfsck_layout |
1776 awk '/^repaired_orphan/ { print $2 }')
1777 [ $repaired -eq $expected ] ||
1778 error "(4) Expect $expected fixed on mds1, but got: $repaired"
1780 if [ $MDSCOUNT -ge 2 ]; then
1781 repaired=$(do_facet mds2 $LCTL get_param -n \
1782 mdd.$(facet_svc mds2).lfsck_layout |
1783 awk '/^repaired_orphan/ { print $2 }')
1784 [ $repaired -eq 0 ] ||
1785 error "(5) Expect 0 fixed on mds2, but got: $repaired"
1788 echo "There should be some stub under .lustre/lost+found/MDT0001/"
1789 ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
1790 error "(6) .lustre/lost+found/MDT0001/ should be empty"
1792 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1793 ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
1794 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1796 run_test 18c "Find out orphan OST-object and repair it (3)"
1800 echo "The target MDT-object layout EA slot is occpuied by some new"
1801 echo "created OST-object when repair dangling reference case. Such"
1802 echo "conflict OST-object has never been modified. Then when found"
1803 echo "the orphan OST-object, LFSCK will replace it with the orphan"
1807 check_mount_and_prep
1809 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1810 echo "guard" > $DIR/$tdir/a1/f1
1811 echo "foo" > $DIR/$tdir/a1/f2
1812 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1813 $LFS path2fid $DIR/$tdir/a1/f1
1814 $LFS getstripe $DIR/$tdir/a1/f1
1815 $LFS path2fid $DIR/$tdir/a1/f2
1816 $LFS getstripe $DIR/$tdir/a1/f2
1817 cancel_lru_locks osc
1819 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1820 echo "to reference the same OST-object (which is f1's OST-obejct)."
1821 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1822 echo "dangling reference case, but f2's old OST-object is there."
1825 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1826 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1827 chown 1.1 $DIR/$tdir/a1/f2
1828 rm -f $DIR/$tdir/a1/f1
1831 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1833 echo "stopall to cleanup object cache"
1836 setupall > /dev/null
1838 echo "The file size should be incorrect since dangling referenced"
1839 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1840 [ "$cur_size" != "$saved_size" ] ||
1841 error "(1) Expect incorrect file2 size"
1843 #define OBD_FAIL_LFSCK_DELAY3 0x1602
1844 do_facet $SINGLEMDS $LCTL set_param fail_val=5 fail_loc=0x1602
1846 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1847 $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
1849 wait_update_facet mds1 "$LCTL get_param -n \
1850 mdd.$(facet_svc mds1).lfsck_layout |
1851 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
1852 error "(3.0) MDS1 is not the expected 'scanning-phase2'"
1854 # LU-3469: before osp_sync() is enabled, wait for a while to guarantee
1855 # that former async repair operations have been executed on the OST(s).
1859 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
1861 for k in $(seq $MDSCOUNT); do
1862 # The LFSCK status query internal is 30 seconds. For the case
1863 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1864 # time to guarantee the status sync up.
1865 wait_update_facet mds${k} "$LCTL get_param -n \
1866 mdd.$(facet_svc mds${k}).lfsck_layout |
1867 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1868 error "(3) MDS${k} is not the expected 'completed'"
1871 for k in $(seq $OSTCOUNT); do
1872 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1873 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1874 awk '/^status/ { print $2 }')
1875 [ "$cur_status" == "completed" ] ||
1876 error "(4) OST${k} Expect 'completed', but got '$cur_status'"
1879 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1880 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1881 awk '/^repaired_orphan/ { print $2 }')
1882 [ $repaired -eq 1 ] ||
1883 error "(5) Expect 1 orphan has been fixed, but got: $repaired"
1885 echo "The file size should be correct after layout LFSCK scanning"
1886 cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1887 [ "$cur_size" == "$saved_size" ] ||
1888 error "(6) Expect file2 size $saved_size, but got $cur_size"
1890 echo "The LFSCK should find back the original data."
1891 cat $DIR/$tdir/a1/f2
1892 $LFS path2fid $DIR/$tdir/a1/f2
1893 $LFS getstripe $DIR/$tdir/a1/f2
1895 run_test 18d "Find out orphan OST-object and repair it (4)"
1899 echo "The target MDT-object layout EA slot is occpuied by some new"
1900 echo "created OST-object when repair dangling reference case. Such"
1901 echo "conflict OST-object has been modified by others. To keep the"
1902 echo "new data, the LFSCK will create a new file to refernece this"
1903 echo "old orphan OST-object."
1906 check_mount_and_prep
1908 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1909 echo "guard" > $DIR/$tdir/a1/f1
1910 echo "foo" > $DIR/$tdir/a1/f2
1911 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1912 $LFS path2fid $DIR/$tdir/a1/f1
1913 $LFS getstripe $DIR/$tdir/a1/f1
1914 $LFS path2fid $DIR/$tdir/a1/f2
1915 $LFS getstripe $DIR/$tdir/a1/f2
1916 cancel_lru_locks osc
1918 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1919 echo "to reference the same OST-object (which is f1's OST-obejct)."
1920 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1921 echo "dangling reference case, but f2's old OST-object is there."
1924 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1925 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1926 chown 1.1 $DIR/$tdir/a1/f2
1927 rm -f $DIR/$tdir/a1/f1
1930 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1932 echo "stopall to cleanup object cache"
1935 setupall > /dev/null
1937 echo "The file size should be incorrect since dangling referenced"
1938 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1939 [ "$cur_size" != "$saved_size" ] ||
1940 error "(1) Expect incorrect file2 size"
1942 #define OBD_FAIL_LFSCK_DELAY3 0x1602
1943 do_facet $SINGLEMDS $LCTL set_param fail_val=10 fail_loc=0x1602
1945 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1946 $START_LAYOUT -r -o -c || error "(2) Fail to start LFSCK for layout!"
1948 wait_update_facet mds1 "$LCTL get_param -n \
1949 mdd.$(facet_svc mds1).lfsck_layout |
1950 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
1951 error "(3) MDS1 is not the expected 'scanning-phase2'"
1953 # to guarantee all updates are synced.
1957 echo "Write new data to f2 to modify the new created OST-object."
1958 echo "dummy" >> $DIR/$tdir/a1/f2
1960 do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0
1962 for k in $(seq $MDSCOUNT); do
1963 # The LFSCK status query internal is 30 seconds. For the case
1964 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1965 # time to guarantee the status sync up.
1966 wait_update_facet mds${k} "$LCTL get_param -n \
1967 mdd.$(facet_svc mds${k}).lfsck_layout |
1968 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1969 error "(4) MDS${k} is not the expected 'completed'"
1972 for k in $(seq $OSTCOUNT); do
1973 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1974 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1975 awk '/^status/ { print $2 }')
1976 [ "$cur_status" == "completed" ] ||
1977 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1980 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1981 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1982 awk '/^repaired_orphan/ { print $2 }')
1983 [ $repaired -eq 1 ] ||
1984 error "(6) Expect 1 orphan has been fixed, but got: $repaired"
1986 echo "There should be stub file under .lustre/lost+found/MDT0000/"
1987 local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
1989 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1991 echo "The stub file should keep the original f2 data"
1992 cur_size=$(ls -il $cname | awk '{ print $6 }')
1993 [ "$cur_size" == "$saved_size" ] ||
1994 error "(8) Expect file2 size $saved_size, but got $cur_size"
1997 $LFS path2fid $cname
1998 $LFS getstripe $cname
2000 echo "The f2 should contains new data."
2001 cat $DIR/$tdir/a1/f2
2002 $LFS path2fid $DIR/$tdir/a1/f2
2003 $LFS getstripe $DIR/$tdir/a1/f2
2005 run_test 18e "Find out orphan OST-object and repair it (5)"
2008 check_mount_and_prep
2009 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2011 echo "foo" > $DIR/$tdir/a0
2012 echo "guard" > $DIR/$tdir/a1
2013 cancel_lru_locks osc
2015 echo "Inject failure, then client will offer wrong parent FID when read"
2016 do_facet ost1 $LCTL set_param -n \
2017 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2018 #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
2019 $LCTL set_param fail_loc=0x1619
2021 echo "Read RPC with wrong parent FID should be denied"
2022 cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
2023 $LCTL set_param fail_loc=0
2025 run_test 19a "OST-object inconsistency self detect"
2028 check_mount_and_prep
2029 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2031 echo "Inject failure stub to make the OST-object to back point to"
2032 echo "non-exist MDT-object"
2034 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
2035 do_facet ost1 $LCTL set_param fail_loc=0x1611
2036 echo "foo" > $DIR/$tdir/f0
2037 cancel_lru_locks osc
2038 do_facet ost1 $LCTL set_param fail_loc=0
2040 echo "Nothing should be fixed since self detect and repair is disabled"
2041 local repaired=$(do_facet ost1 $LCTL get_param -n \
2042 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2043 awk '/^repaired/ { print $2 }')
2044 [ $repaired -eq 0 ] ||
2045 error "(1) Expected 0 repaired, but got $repaired"
2047 echo "Read RPC with right parent FID should be accepted,"
2048 echo "and cause parent FID on OST to be fixed"
2050 do_facet ost1 $LCTL set_param -n \
2051 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2052 cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
2054 repaired=$(do_facet ost1 $LCTL get_param -n \
2055 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2056 awk '/^repaired/ { print $2 }')
2057 [ $repaired -eq 1 ] ||
2058 error "(3) Expected 1 repaired, but got $repaired"
2060 run_test 19b "OST-object inconsistency self repair"
2062 $LCTL set_param debug=-lfsck > /dev/null || true
2064 # restore MDS/OST size
2065 MDSSIZE=${SAVED_MDSSIZE}
2066 OSTSIZE=${SAVED_OSTSIZE}
2067 OSTCOUNT=${SAVED_OSTCOUNT}
2069 # cleanup the system at last