3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 MCREATE=${MCREATE:-mcreate}
23 SAVED_MDSSIZE=${MDSSIZE}
24 SAVED_OSTSIZE=${OSTSIZE}
25 # use small MDS + OST size to speed formatting time
26 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
33 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
36 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
37 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
39 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
44 $LCTL set_param debug=+lfsck > /dev/null || true
46 MDT_DEV="${FSNAME}-MDT0000"
47 OST_DEV="${FSNAME}-OST0000"
48 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
49 START_NAMESPACE="do_facet $SINGLEMDS \
50 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
51 START_LAYOUT="do_facet $SINGLEMDS \
52 $LCTL lfsck_start -M ${MDT_DEV} -t layout"
53 START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
54 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
55 SHOW_NAMESPACE="do_facet $SINGLEMDS \
56 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
57 SHOW_LAYOUT="do_facet $SINGLEMDS \
58 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
59 SHOW_LAYOUT_ON_OST="do_facet ost1 \
60 $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
61 MOUNT_OPTS_SCRUB="-o user_xattr"
62 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
75 if [ ! -z $igif ]; then
76 #define OBD_FAIL_FID_IGIF 0x1504
77 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
80 echo "preparing... ${nfiles} * ${ndirs} files will be created."
82 cp $LUSTRE/tests/*.sh $DIR/
83 for ((i = 0; i < ${ndirs}; i++)); do
84 mkdir $DIR/$tdir/d${i}
85 touch $DIR/$tdir/f${i}
86 for ((j = 0; j < ${nfiles}; j++)); do
87 touch $DIR/$tdir/d${i}/f${j}
89 mkdir $DIR/$tdir/e${i}
92 if [ ! -z $igif ]; then
93 touch $DIR/$tdir/dummy
94 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
98 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
99 echo "stop $SINGLEMDS"
100 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
105 echo "start $SINGLEMDS"
106 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
107 error "(1) Fail to start MDS!"
109 #define OBD_FAIL_LFSCK_DELAY1 0x1600
110 do_facet $SINGLEMDS $LCTL set_param fail_val=3
111 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
112 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
114 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
116 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
117 [ "$STATUS" == "scanning-phase1" ] ||
118 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
120 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
122 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
123 [ "$STATUS" == "stopped" ] ||
124 error "(6) Expect 'stopped', but got '$STATUS'"
126 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
128 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
129 [ "$STATUS" == "scanning-phase1" ] ||
130 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
132 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
133 do_facet $SINGLEMDS $LCTL set_param fail_val=0
134 wait_update_facet $SINGLEMDS \
135 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
136 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
137 error "(9) unexpected status"
139 local repaired=$($SHOW_NAMESPACE |
140 awk '/^updated_phase1/ { print $2 }')
141 [ $repaired -eq 0 ] ||
142 error "(10) Expect nothing to be repaired, but got: $repaired"
144 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
145 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
146 wait_update_facet $SINGLEMDS \
147 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
148 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
149 error "(12) unexpected status"
151 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
152 [ $((scanned1 + 1)) -eq $scanned2 ] ||
153 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
155 echo "stopall, should NOT crash LU-3649"
158 run_test 0 "Control LFSCK manually"
161 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
162 skip "OI Scrub not implemented for ZFS" && return
165 echo "start $SINGLEMDS"
166 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
167 error "(1) Fail to start MDS!"
169 mount_client $MOUNT || error "(2) Fail to start client!"
171 #define OBD_FAIL_FID_INDIR 0x1501
172 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
173 touch $DIR/$tdir/dummy
175 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
177 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
180 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
181 [ "$STATUS" == "completed" ] ||
182 error "(4) Expect 'completed', but got '$STATUS'"
184 local repaired=$($SHOW_NAMESPACE |
185 awk '/^updated_phase1/ { print $2 }')
186 [ $repaired -eq 1 ] ||
187 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
189 mount_client $MOUNT || error "(6) Fail to start client!"
191 #define OBD_FAIL_FID_LOOKUP 0x1505
192 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
193 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
195 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
197 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
201 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
202 skip "OI Scrub not implemented for ZFS" && return
205 echo "start $SINGLEMDS"
206 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
207 error "(1) Fail to start MDS!"
209 mount_client $MOUNT || error "(2) Fail to start client!"
211 #define OBD_FAIL_FID_INLMA 0x1502
212 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
213 touch $DIR/$tdir/dummy
215 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
217 #define OBD_FAIL_FID_NOLMA 0x1506
218 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
219 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
222 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
223 [ "$STATUS" == "completed" ] ||
224 error "(4) Expect 'completed', but got '$STATUS'"
226 local repaired=$($SHOW_NAMESPACE |
227 awk '/^updated_phase1/ { print $2 }')
228 [ $repaired -eq 1 ] ||
229 error "(5) Fail to repair missed FID-in-LMA: $repaired"
231 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
232 mount_client $MOUNT || error "(6) Fail to start client!"
234 #define OBD_FAIL_FID_LOOKUP 0x1505
235 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
236 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
238 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
240 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
244 echo "start $SINGLEMDS"
245 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
246 error "(1) Fail to start MDS!"
248 mount_client $MOUNT || error "(2) Fail to start client!"
250 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
251 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
252 touch $DIR/$tdir/dummy
254 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
256 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
259 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
260 [ "$STATUS" == "completed" ] ||
261 error "(4) Expect 'completed', but got '$STATUS'"
263 local repaired=$($SHOW_NAMESPACE |
264 awk '/^updated_phase1/ { print $2 }')
265 [ $repaired -eq 1 ] ||
266 error "(5) Fail to repair crashed linkEA: $repaired"
268 mount_client $MOUNT || error "(6) Fail to start client!"
270 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
271 error "(7) Fail to stat $DIR/$tdir/dummy"
273 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
274 local dummyname=$($LFS fid2path $DIR $dummyfid)
275 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
276 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
278 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
283 echo "start $SINGLEMDS"
284 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
285 error "(1) Fail to start MDS!"
287 mount_client $MOUNT || error "(2) Fail to start client!"
289 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
290 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
291 touch $DIR/$tdir/dummy
293 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
295 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
298 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
299 [ "$STATUS" == "completed" ] ||
300 error "(4) Expect 'completed', but got '$STATUS'"
302 local repaired=$($SHOW_NAMESPACE |
303 awk '/^updated_phase2/ { print $2 }')
304 [ $repaired -eq 1 ] ||
305 error "(5) Fail to repair crashed linkEA: $repaired"
307 mount_client $MOUNT || error "(6) Fail to start client!"
309 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
310 error "(7) Fail to stat $DIR/$tdir/dummy"
312 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
313 local dummyname=$($LFS fid2path $DIR $dummyfid)
314 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
315 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
317 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
322 echo "start $SINGLEMDS"
323 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
324 error "(1) Fail to start MDS!"
326 mount_client $MOUNT || error "(2) Fail to start client!"
328 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
330 touch $DIR/$tdir/dummy
332 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
334 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
337 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
338 [ "$STATUS" == "completed" ] ||
339 error "(4) Expect 'completed', but got '$STATUS'"
341 local repaired=$($SHOW_NAMESPACE |
342 awk '/^updated_phase2/ { print $2 }')
343 [ $repaired -eq 1 ] ||
344 error "(5) Fail to repair crashed linkEA: $repaired"
346 mount_client $MOUNT || error "(6) Fail to start client!"
348 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
349 error "(7) Fail to stat $DIR/$tdir/dummy"
351 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
352 local dummyname=$($LFS fid2path $DIR $dummyfid)
353 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
354 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
356 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
360 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
361 skip "OI Scrub not implemented for ZFS" && return
364 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
365 echo "start $SINGLEMDS with disabling OI scrub"
366 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
367 error "(2) Fail to start MDS!"
369 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
370 [ "$STATUS" == "init" ] ||
371 error "(3) Expect 'init', but got '$STATUS'"
373 #define OBD_FAIL_LFSCK_DELAY2 0x1601
374 do_facet $SINGLEMDS $LCTL set_param fail_val=1
375 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
376 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
379 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
380 [ "$STATUS" == "scanning-phase1" ] ||
381 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
383 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
384 [ "$FLAGS" == "inconsistent" ] ||
385 error "(6) Expect 'inconsistent', but got '$FLAGS'"
387 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
388 do_facet $SINGLEMDS $LCTL set_param fail_val=0
390 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
391 [ "$STATUS" == "completed" ] ||
392 error "(7) Expect 'completed', but got '$STATUS'"
394 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
395 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
397 local repaired=$($SHOW_NAMESPACE |
398 awk '/^updated_phase1/ { print $2 }')
399 [ $repaired -ge 9 ] ||
400 error "(9) Fail to repair crashed linkEA: $repaired"
402 mount_client $MOUNT || error "(10) Fail to start client!"
404 #define OBD_FAIL_FID_LOOKUP 0x1505
405 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
406 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
408 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
410 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
414 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
415 skip "OI Scrub not implemented for ZFS" && return
418 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
419 echo "start $SINGLEMDS with disabling OI scrub"
420 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
421 error "(2) Fail to start MDS!"
423 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
424 [ "$STATUS" == "init" ] ||
425 error "(3) Expect 'init', but got '$STATUS'"
427 #define OBD_FAIL_LFSCK_DELAY2 0x1601
428 do_facet $SINGLEMDS $LCTL set_param fail_val=1
429 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
430 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
433 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
434 [ "$STATUS" == "scanning-phase1" ] ||
435 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
437 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
438 [ "$FLAGS" == "inconsistent,upgrade" ] ||
439 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
441 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
442 do_facet $SINGLEMDS $LCTL set_param fail_val=0
444 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
445 [ "$STATUS" == "completed" ] ||
446 error "(7) Expect 'completed', but got '$STATUS'"
448 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
449 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
451 local repaired=$($SHOW_NAMESPACE |
452 awk '/^updated_phase1/ { print $2 }')
453 [ $repaired -ge 2 ] ||
454 error "(9) Fail to repair crashed linkEA: $repaired"
456 mount_client $MOUNT || error "(10) Fail to start client!"
458 #define OBD_FAIL_FID_LOOKUP 0x1505
459 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
460 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
462 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
464 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
465 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
466 local dummyname=$($LFS fid2path $DIR $dummyfid)
467 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
468 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
470 run_test 5 "LFSCK can handle IFIG object upgrading"
474 echo "start $SINGLEMDS"
475 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
476 error "(1) Fail to start MDS!"
478 #define OBD_FAIL_LFSCK_DELAY1 0x1600
479 do_facet $SINGLEMDS $LCTL set_param fail_val=1
480 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
481 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
483 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
484 [ "$STATUS" == "scanning-phase1" ] ||
485 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
487 # Sleep 3 sec to guarantee at least one object processed by LFSCK
489 # Fail the LFSCK to guarantee there is at least one checkpoint
490 #define OBD_FAIL_LFSCK_FATAL1 0x1608
491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
493 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
494 [ "$STATUS" == "failed" ] ||
495 error "(4) Expect 'failed', but got '$STATUS'"
497 local POSITION0=$($SHOW_NAMESPACE |
498 awk '/^last_checkpoint_position/ { print $2 }' |
501 #define OBD_FAIL_LFSCK_DELAY1 0x1600
502 do_facet $SINGLEMDS $LCTL set_param fail_val=1
503 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
504 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
506 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
507 [ "$STATUS" == "scanning-phase1" ] ||
508 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
510 local POSITION1=$($SHOW_NAMESPACE |
511 awk '/^latest_start_position/ { print $2 }' |
513 [ $POSITION0 -lt $POSITION1 ] ||
514 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
516 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
517 do_facet $SINGLEMDS $LCTL set_param fail_val=0
519 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
520 [ "$STATUS" == "completed" ] ||
521 error "(8) Expect 'completed', but got '$STATUS'"
523 run_test 6a "LFSCK resumes from last checkpoint (1)"
527 echo "start $SINGLEMDS"
528 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
529 error "(1) Fail to start MDS!"
531 #define OBD_FAIL_LFSCK_DELAY2 0x1601
532 do_facet $SINGLEMDS $LCTL set_param fail_val=1
533 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
534 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
536 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
537 [ "$STATUS" == "scanning-phase1" ] ||
538 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
540 # Sleep 3 sec to guarantee at least one object processed by LFSCK
542 # Fail the LFSCK to guarantee there is at least one checkpoint
543 #define OBD_FAIL_LFSCK_FATAL2 0x1609
544 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
546 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
547 [ "$STATUS" == "failed" ] ||
548 error "(4) Expect 'failed', but got '$STATUS'"
550 local POSITION0=$($SHOW_NAMESPACE |
551 awk '/^last_checkpoint_position/ { print $4 }')
553 #define OBD_FAIL_LFSCK_DELAY2 0x1601
554 do_facet $SINGLEMDS $LCTL set_param fail_val=1
555 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
556 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
558 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
559 [ "$STATUS" == "scanning-phase1" ] ||
560 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
562 local POSITION1=$($SHOW_NAMESPACE |
563 awk '/^latest_start_position/ { print $4 }')
564 if [ $POSITION0 -gt $POSITION1 ]; then
565 [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
566 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
569 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
570 do_facet $SINGLEMDS $LCTL set_param fail_val=0
572 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "completed" ] ||
574 error "(8) Expect 'completed', but got '$STATUS'"
576 run_test 6b "LFSCK resumes from last checkpoint (2)"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(1) Fail to start MDS!"
585 #define OBD_FAIL_LFSCK_DELAY2 0x1601
586 do_facet $SINGLEMDS $LCTL set_param fail_val=1
587 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
588 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
590 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
591 [ "$STATUS" == "scanning-phase1" ] ||
592 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
594 # Sleep 3 sec to guarantee at least one object processed by LFSCK
596 echo "stop $SINGLEMDS"
597 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
599 echo "start $SINGLEMDS"
600 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
601 error "(5) Fail to start MDS!"
603 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
604 [ "$STATUS" == "scanning-phase1" ] ||
605 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
607 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
608 do_facet $SINGLEMDS $LCTL set_param fail_val=0
610 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
611 [ "$STATUS" == "completed" ] ||
612 error "(7) Expect 'completed', but got '$STATUS'"
614 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
619 echo "start $SINGLEMDS"
620 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
621 error "(1) Fail to start MDS!"
623 mount_client $MOUNT || error "(2) Fail to start client!"
625 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
626 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
627 for ((i = 0; i < 20; i++)); do
628 touch $DIR/$tdir/dummy${i}
631 #define OBD_FAIL_LFSCK_DELAY3 0x1602
632 do_facet $SINGLEMDS $LCTL set_param fail_val=1
633 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
634 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
637 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
638 [ "$STATUS" == "scanning-phase2" ] ||
639 error "(4) Expect 'scanning-phase2', but got '$STATUS'"
641 echo "stop $SINGLEMDS"
642 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
644 echo "start $SINGLEMDS"
645 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
646 error "(6) Fail to start MDS!"
648 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "scanning-phase2" ] ||
650 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
652 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
653 do_facet $SINGLEMDS $LCTL set_param fail_val=0
655 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
656 [ "$STATUS" == "completed" ] ||
657 error "(8) Expect 'completed', but got '$STATUS'"
659 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
664 echo "start $SINGLEMDS"
665 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
666 error "(1) Fail to start MDS!"
668 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
669 [ "$STATUS" == "init" ] ||
670 error "(2) Expect 'init', but got '$STATUS'"
672 mount_client $MOUNT || error "(3) Fail to start client!"
674 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
675 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
676 mkdir $DIR/$tdir/crashed
678 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
679 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
680 for ((i = 0; i < 5; i++)); do
681 touch $DIR/$tdir/dummy${i}
684 #define OBD_FAIL_LFSCK_DELAY2 0x1601
685 do_facet $SINGLEMDS $LCTL set_param fail_val=2
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
687 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
689 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
690 [ "$STATUS" == "scanning-phase1" ] ||
691 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
693 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
695 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
696 [ "$STATUS" == "stopped" ] ||
697 error "(7) Expect 'stopped', but got '$STATUS'"
699 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
701 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
702 [ "$STATUS" == "scanning-phase1" ] ||
703 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
705 #define OBD_FAIL_LFSCK_FATAL2 0x1609
706 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
708 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
709 [ "$STATUS" == "failed" ] ||
710 error "(10) Expect 'failed', but got '$STATUS'"
712 #define OBD_FAIL_LFSCK_DELAY1 0x1600
713 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
714 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
716 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "scanning-phase1" ] ||
718 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
720 #define OBD_FAIL_LFSCK_CRASH 0x160a
721 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
724 echo "stop $SINGLEMDS"
725 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
727 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
728 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
730 echo "start $SINGLEMDS"
731 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
732 error "(14) Fail to start MDS!"
734 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
735 [ "$STATUS" == "crashed" ] ||
736 error "(15) Expect 'crashed', but got '$STATUS'"
738 #define OBD_FAIL_LFSCK_DELAY2 0x1601
739 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
740 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
742 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
743 [ "$STATUS" == "scanning-phase1" ] ||
744 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
746 echo "stop $SINGLEMDS"
747 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
749 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
750 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
752 echo "start $SINGLEMDS"
753 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
754 error "(19) Fail to start MDS!"
756 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
757 [ "$STATUS" == "paused" ] ||
758 error "(20) Expect 'paused', but got '$STATUS'"
760 #define OBD_FAIL_LFSCK_DELAY3 0x1602
761 do_facet $SINGLEMDS $LCTL set_param fail_val=2
762 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
764 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
766 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
767 [ "$STATUS" == "scanning-phase2" ] ||
768 error "(22) Expect 'scanning-phase2', but got '$STATUS'"
770 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
771 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
772 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
774 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
775 do_facet $SINGLEMDS $LCTL set_param fail_val=0
777 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
778 [ "$STATUS" == "completed" ] ||
779 error "(24) Expect 'completed', but got '$STATUS'"
781 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
782 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
785 run_test 8 "LFSCK state machine"
788 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
789 skip "Testing on UP system, the speed may be inaccurate."
794 echo "start $SINGLEMDS"
795 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
796 error "(1) Fail to start MDS!"
798 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
799 [ "$STATUS" == "init" ] ||
800 error "(2) Expect 'init', but got '$STATUS'"
802 local BASE_SPEED1=100
804 $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
807 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
808 [ "$STATUS" == "scanning-phase1" ] ||
809 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
811 local SPEED=$($SHOW_NAMESPACE |
812 awk '/^average_speed_phase1/ { print $2 }')
814 # There may be time error, normally it should be less than 2 seconds.
815 # We allow another 20% schedule error.
817 # MAX_MARGIN = 1.2 = 12 / 10
818 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
819 RUN_TIME1 * 12 / 10))
820 [ $SPEED -lt $MAX_SPEED ] ||
821 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
824 local BASE_SPEED2=300
826 do_facet $SINGLEMDS \
827 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
830 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
831 # MIN_MARGIN = 0.8 = 8 / 10
832 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
833 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
834 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
835 [ $SPEED -gt $MIN_SPEED ] ||
836 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
838 # MAX_MARGIN = 1.2 = 12 / 10
839 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
840 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
841 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
842 [ $SPEED -lt $MAX_SPEED ] ||
843 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
845 do_facet $SINGLEMDS \
846 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
848 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
849 [ "$STATUS" == "completed" ] ||
850 error "(7) Expect 'completed', but got '$STATUS'"
852 run_test 9a "LFSCK speed control (1)"
855 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
856 skip "Testing on UP system, the speed may be inaccurate."
861 echo "start $SINGLEMDS"
862 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
863 error "(1) Fail to start MDS!"
865 mount_client $MOUNT || error "(2) Fail to start client!"
867 echo "Another preparing... 50 * 50 files (with error) will be created."
868 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
869 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
870 for ((i = 0; i < 50; i++)); do
871 mkdir -p $DIR/$tdir/d${i}
872 touch $DIR/$tdir/f${i}
873 for ((j = 0; j < 50; j++)); do
874 touch $DIR/$tdir/d${i}/f${j}
878 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
879 [ "$STATUS" == "init" ] ||
880 error "(3) Expect 'init', but got '$STATUS'"
882 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
883 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
884 $START_NAMESPACE || error "(4) Fail to start LFSCK!"
887 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
888 [ "$STATUS" == "stopped" ] ||
889 error "(5) Expect 'stopped', but got '$STATUS'"
891 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
895 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
898 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
899 [ "$STATUS" == "scanning-phase2" ] ||
900 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
902 local SPEED=$($SHOW_NAMESPACE |
903 awk '/^average_speed_phase2/ { print $2 }')
904 # There may be time error, normally it should be less than 2 seconds.
905 # We allow another 20% schedule error.
907 # MAX_MARGIN = 1.2 = 12 / 10
908 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
909 RUN_TIME1 * 12 / 10))
910 [ $SPEED -lt $MAX_SPEED ] ||
911 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
914 local BASE_SPEED2=150
916 do_facet $SINGLEMDS \
917 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
920 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
921 # MIN_MARGIN = 0.8 = 8 / 10
922 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
923 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
924 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
925 [ $SPEED -gt $MIN_SPEED ] ||
926 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
928 # MAX_MARGIN = 1.2 = 12 / 10
929 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
930 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
931 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
932 [ $SPEED -lt $MAX_SPEED ] ||
933 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
935 do_facet $SINGLEMDS \
936 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
938 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
939 [ "$STATUS" == "completed" ] ||
940 error "(11) Expect 'completed', but got '$STATUS'"
942 run_test 9b "LFSCK speed control (2)"
946 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
947 skip "lookup(..)/linkea on ZFS issue" && return
950 echo "start $SINGLEMDS"
951 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
952 error "(1) Fail to start MDS!"
954 mount_client $MOUNT || error "(2) Fail to start client!"
956 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
957 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
958 for ((i = 0; i < 1000; i = $((i+2)))); do
959 mkdir -p $DIR/$tdir/d${i}
960 touch $DIR/$tdir/f${i}
961 for ((j = 0; j < 5; j++)); do
962 touch $DIR/$tdir/d${i}/f${j}
966 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
967 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
968 for ((i = 1; i < 1000; i = $((i+2)))); do
969 mkdir -p $DIR/$tdir/d${i}
970 touch $DIR/$tdir/f${i}
971 for ((j = 0; j < 5; j++)); do
972 touch $DIR/$tdir/d${i}/f${j}
976 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
977 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
980 mount_client $MOUNT || error "(3) Fail to start client!"
982 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
983 [ "$STATUS" == "init" ] ||
984 error "(4) Expect 'init', but got '$STATUS'"
986 $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
989 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
990 [ "$STATUS" == "scanning-phase1" ] ||
991 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
993 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
995 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
997 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
999 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
1001 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
1003 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
1005 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
1007 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
1008 error "(14) Fail to softlink!"
1010 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1011 [ "$STATUS" == "scanning-phase1" ] ||
1012 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
1014 do_facet $SINGLEMDS \
1015 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
1016 umount_client $MOUNT
1018 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1019 [ "$STATUS" == "completed" ] ||
1020 error "(16) Expect 'completed', but got '$STATUS'"
1022 run_test 10 "System is available during LFSCK scanning"
1025 ost_remove_lastid() {
1028 local rcmd="do_facet ost${ost}"
1030 echo "remove LAST_ID on ost${ost}: idx=${idx}"
1032 # step 1: local mount
1033 mount_fstype ost${ost} || return 1
1034 # step 2: remove the specified LAST_ID
1035 ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/{LAST_ID,d0/0}
1037 unmount_fstype ost${ost} || return 2
1044 formatall > /dev/null
1046 setupall > /dev/null
1049 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1050 createmany -o $DIR/$tdir/f 64
1055 ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
1058 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
1059 error "(2) Fail to start ost1"
1061 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1062 [ "$STATUS" == "init" ] ||
1063 error "(3) Expect 'init', but got '$STATUS'"
1065 #define OBD_FAIL_LFSCK_DELAY4 0x160e
1066 do_facet ost1 $LCTL set_param fail_val=3
1067 do_facet ost1 $LCTL set_param fail_loc=0x160e
1069 echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
1070 $START_LAYOUT_ON_OST || error "(4) Fail to start LFSCK on OST!"
1072 wait_update_facet ost1 "$LCTL get_param -n \
1073 obdfilter.${OST_DEV}.lfsck_layout |
1074 awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
1079 do_facet ost1 $LCTL set_param fail_val=0
1080 do_facet ost1 $LCTL set_param fail_loc=0
1082 wait_update_facet ost1 "$LCTL get_param -n \
1083 obdfilter.${OST_DEV}.lfsck_layout |
1084 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1089 echo "the LAST_ID(s) should have been rebuilt"
1090 FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
1091 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
1093 run_test 11a "LFSCK can rebuild lost last_id"
1099 formatall > /dev/null
1101 setupall > /dev/null
1104 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1106 echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
1107 #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
1108 do_facet ost1 $LCTL set_param fail_loc=0x160d
1109 createmany -o $DIR/$tdir/f 64
1110 local lastid1=$(do_facet ost1 "lctl get_param -n \
1111 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1112 awk -F: '{ print $2 }')
1114 umount_client $MOUNT
1116 stop ost1 || error "(1) Fail to stop ost1"
1118 #define OBD_FAIL_OST_ENOSPC 0x215
1119 do_facet ost1 $LCTL set_param fail_loc=0x215
1122 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1123 error "(2) Fail to start ost1"
1125 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1126 [ "$STATUS" == "init" ] ||
1127 error "(3) Expect 'init', but got '$STATUS'"
1129 for ((i = 0; i < 60; i++)); do
1130 lastid2=$(do_facet ost1 "lctl get_param -n \
1131 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1132 awk -F: '{ print $2 }')
1133 [ ! -z $lastid2 ] && break;
1137 echo "the on-disk LAST_ID should be smaller than the expected one"
1138 [ $lastid1 -gt $lastid2 ] ||
1139 error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
1141 echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
1142 $START_LAYOUT_ON_OST || error "(5) Fail to start LFSCK on OST!"
1144 wait_update_facet ost1 "$LCTL get_param -n \
1145 obdfilter.${OST_DEV}.lfsck_layout |
1146 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1152 stop ost1 || error "(7) Fail to stop ost1"
1155 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1156 error "(8) Fail to start ost1"
1158 echo "the on-disk LAST_ID should have been rebuilt"
1159 wait_update_facet ost1 "$LCTL get_param -n \
1160 obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
1161 awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
1162 $LCTL get_param -n obdfilter.${ost1_svc}.last_id
1163 error "(9) expect lastid1 0x100000000:$lastid1"
1166 do_facet ost1 $LCTL set_param fail_loc=0
1168 run_test 11b "LFSCK can rebuild crashed last_id"
1171 [ $MDSCOUNT -lt 2 ] &&
1172 skip "We need at least 2 MDSes for test_12" && exit 0
1177 formatall > /dev/null
1179 setupall > /dev/null
1183 echo "All the LFSCK targets should be in 'init' status."
1184 for k in $(seq $MDSCOUNT); do
1185 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1186 mdd.$(facet_svc mds${k}).lfsck_layout |
1187 awk '/^status/ { print $2 }')
1188 [ "$STATUS" == "init" ] ||
1189 error "(1) MDS${k} Expect 'init', but got '$STATUS'"
1191 $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
1192 createmany -o $DIR/$tdir/${k}/f 100
1195 echo "Start namespace LFSCK on all targets by single command (-s 1)."
1196 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1197 -s 1 || error "(2) Fail to start LFSCK on all devices!"
1199 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1200 for k in $(seq $MDSCOUNT); do
1201 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1202 mdd.$(facet_svc mds${k}).lfsck_namespace |
1203 awk '/^status/ { print $2 }')
1204 [ "$STATUS" == "scanning-phase1" ] ||
1205 error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1208 echo "Stop namespace LFSCK on all targets by single lctl command."
1209 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1210 error "(4) Fail to stop LFSCK on all devices!"
1212 echo "All the LFSCK targets should be in 'stopped' status."
1213 for k in $(seq $MDSCOUNT); do
1214 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1215 mdd.$(facet_svc mds${k}).lfsck_namespace |
1216 awk '/^status/ { print $2 }')
1217 [ "$STATUS" == "stopped" ] ||
1218 error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
1221 echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
1222 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1223 -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
1225 echo "All the LFSCK targets should be in 'completed' status."
1226 for k in $(seq $MDSCOUNT); do
1227 wait_update_facet mds${k} "$LCTL get_param -n \
1228 mdd.$(facet_svc mds${k}).lfsck_namespace |
1229 awk '/^status/ { print \\\$2 }'" "completed" 8 ||
1230 error "(7) MDS${k} is not the expected 'completed'"
1233 echo "Start layout LFSCK on all targets by single command (-s 1)."
1234 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1235 -s 1 || error "(8) Fail to start LFSCK on all devices!"
1237 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1238 for k in $(seq $MDSCOUNT); do
1239 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1240 mdd.$(facet_svc mds${k}).lfsck_layout |
1241 awk '/^status/ { print $2 }')
1242 [ "$STATUS" == "scanning-phase1" ] ||
1243 error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1246 echo "Stop layout LFSCK on all targets by single lctl command."
1247 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1248 error "(10) Fail to stop LFSCK on all devices!"
1250 echo "All the LFSCK targets should be in 'stopped' status."
1251 for k in $(seq $MDSCOUNT); do
1252 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1253 mdd.$(facet_svc mds${k}).lfsck_layout |
1254 awk '/^status/ { print $2 }')
1255 [ "$STATUS" == "stopped" ] ||
1256 error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
1259 for k in $(seq $OSTCOUNT); do
1260 local STATUS=$(do_facet ost${k} $LCTL get_param -n \
1261 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1262 awk '/^status/ { print $2 }')
1263 [ "$STATUS" == "stopped" ] ||
1264 error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
1267 echo "Re-start layout LFSCK on all targets by single command (-s 0)."
1268 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1269 -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
1271 echo "All the LFSCK targets should be in 'completed' status."
1272 for k in $(seq $MDSCOUNT); do
1273 # The LFSCK status query internal is 30 seconds. For the case
1274 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1275 # time to guarantee the status sync up.
1276 wait_update_facet mds${k} "$LCTL get_param -n \
1277 mdd.$(facet_svc mds${k}).lfsck_layout |
1278 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1279 error "(14) MDS${k} is not the expected 'completed'"
1282 run_test 12 "single command to trigger LFSCK on all devices"
1286 echo "The lmm_oi in layout EA should be consistent with the MDT-object"
1287 echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
1288 echo "MDT-object FID."
1294 formatall > /dev/null
1296 setupall > /dev/null
1300 echo "Inject failure stub to simulate bad lmm_oi"
1301 #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
1302 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
1303 createmany -o $DIR/$tdir/f 32
1304 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1306 echo "stopall to cleanup object cache"
1309 setupall > /dev/null
1311 echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
1312 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1314 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1315 mdd.${MDT_DEV}.lfsck_layout |
1316 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1318 local repaired=$($SHOW_LAYOUT |
1319 awk '/^repaired_others/ { print $2 }')
1320 [ $repaired -eq 32 ] ||
1321 error "(3) Fail to repair crashed lmm_oi: $repaired"
1323 run_test 13 "LFSCK can repair crashed lmm_oi"
1327 echo "The OST-object referenced by the MDT-object should be there;"
1328 echo "otherwise, the LFSCK should re-create the missed OST-object."
1334 formatall > /dev/null
1336 setupall > /dev/null
1339 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1341 echo "Inject failure stub to simulate dangling referenced MDT-object"
1342 #define OBD_FAIL_LFSCK_DANGLING 0x1610
1343 do_facet ost1 $LCTL set_param fail_loc=0x1610
1344 createmany -o $DIR/$tdir/f 64
1345 do_facet ost1 $LCTL set_param fail_loc=0
1347 echo "stopall to cleanup object cache"
1350 setupall > /dev/null
1352 echo "'ls' should fail because of dangling referenced MDT-object"
1353 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
1355 echo "Trigger layout LFSCK to find out dangling reference and fix them"
1356 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1358 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1359 mdd.${MDT_DEV}.lfsck_layout |
1360 awk '/^status/ { print \\\$2 }'" "completed" 6 || return 3
1362 local repaired=$($SHOW_LAYOUT |
1363 awk '/^repaired_dangling/ { print $2 }')
1364 [ $repaired -eq 32 ] ||
1365 error "(4) Fail to repair dangling reference: $repaired"
1367 echo "'ls' should success after layout LFSCK repairing"
1368 ls -ail $DIR/$tdir > /dev/null || error "(5) ls should success."
1370 run_test 14 "LFSCK can repair MDT-object with dangling reference"
1374 echo "If the OST-object referenced by the MDT-object back points"
1375 echo "to some non-exist MDT-object, then the LFSCK should repair"
1376 echo "the OST-object to back point to the right MDT-object."
1382 formatall > /dev/null
1384 setupall > /dev/null
1387 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1389 echo "Inject failure stub to make the OST-object to back point to"
1390 echo "non-exist MDT-object."
1391 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
1393 do_facet ost1 $LCTL set_param fail_loc=0x1611
1394 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1395 cancel_lru_locks osc
1398 do_facet ost1 $LCTL set_param fail_loc=0
1400 echo "stopall to cleanup object cache"
1403 setupall > /dev/null
1405 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1406 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1408 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1409 mdd.${MDT_DEV}.lfsck_layout |
1410 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1412 local repaired=$($SHOW_LAYOUT |
1413 awk '/^repaired_unmatched_pair/ { print $2 }')
1414 [ $repaired -eq 1 ] ||
1415 error "(3) Fail to repair unmatched pair: $repaired"
1417 run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
1421 echo "If the OST-object referenced by the MDT-object back points"
1422 echo "to other MDT-object that doesn't recognize the OST-object,"
1423 echo "then the LFSCK should repair it to back point to the right"
1424 echo "MDT-object (the first one)."
1430 formatall > /dev/null
1432 setupall > /dev/null
1435 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1436 touch $DIR/$tdir/guard
1438 echo "Inject failure stub to make the OST-object to back point to"
1439 echo "other MDT-object"
1441 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
1442 do_facet ost1 $LCTL set_param fail_loc=0x1612
1443 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1444 cancel_lru_locks osc
1447 do_facet ost1 $LCTL set_param fail_loc=0
1449 echo "stopall to cleanup object cache"
1452 setupall > /dev/null
1454 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1455 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1457 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1458 mdd.${MDT_DEV}.lfsck_layout |
1459 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1461 local repaired=$($SHOW_LAYOUT |
1462 awk '/^repaired_unmatched_pair/ { print $2 }')
1463 [ $repaired -eq 1 ] ||
1464 error "(3) Fail to repair unmatched pair: $repaired"
1466 run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
1470 echo "If the OST-object's owner information does not match the owner"
1471 echo "information stored in the MDT-object, then the LFSCK trust the"
1472 echo "MDT-object and update the OST-object's owner information."
1478 formatall > /dev/null
1480 setupall > /dev/null
1483 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1484 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1485 cancel_lru_locks osc
1489 echo "Inject failure stub to skip OST-object owner changing"
1490 #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
1491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
1492 chown 1.1 $DIR/$tdir/f0
1493 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1495 echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
1498 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1500 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1501 mdd.${MDT_DEV}.lfsck_layout |
1502 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1504 local repaired=$($SHOW_LAYOUT |
1505 awk '/^repaired_inconsistent_owner/ { print $2 }')
1506 [ $repaired -eq 1 ] ||
1507 error "(3) Fail to repair inconsistent owner: $repaired"
1509 run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
1513 echo "If more than one MDT-objects reference the same OST-object,"
1514 echo "and the OST-object only recognizes one MDT-object, then the"
1515 echo "LFSCK should create new OST-objects for such non-recognized"
1522 formatall > /dev/null
1524 setupall > /dev/null
1527 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1529 echo "Inject failure stub to make two MDT-objects to refernce"
1530 echo "the OST-object"
1532 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1533 #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
1534 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1614
1536 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1537 cancel_lru_locks osc
1541 createmany -o $DIR/$tdir/f 1 > /dev/null 2>&1
1543 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1544 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1546 echo "stopall to cleanup object cache"
1549 setupall > /dev/null
1551 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
1552 local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
1553 [ $size -eq 1048576 ] ||
1554 error "(1) f0 (wrong) size should be 1048576, but got $size"
1556 echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
1559 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1561 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1562 mdd.${MDT_DEV}.lfsck_layout |
1563 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 3
1565 local repaired=$($SHOW_LAYOUT |
1566 awk '/^repaired_multiple_referenced/ { print $2 }')
1567 [ $repaired -eq 1 ] ||
1568 error "(4) Fail to repair multiple references: $repaired"
1570 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
1571 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
1572 error "(5) Fail to write f0."
1573 size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
1574 [ $size -eq 1048576 ] ||
1575 error "(6) guard size should be 1048576, but got $size"
1577 run_test 17 "LFSCK can repair multiple references"
1580 [ $MDSCOUNT -lt 2 ] &&
1581 skip "We need at least 2 MDSes for test_18a" && exit 0
1583 [ $OSTCOUNT -lt 2 ] &&
1584 skip "We need at least 2 OSTs for test_18a" && exit 0
1587 echo "The target MDT-object is there, but related stripe information"
1588 echo "is lost or partly lost. The LFSCK should regenerate the missed"
1589 echo "layout EA entries."
1595 formatall > /dev/null
1597 setupall > /dev/null
1600 $LFS mkdir -i 0 $DIR/$tdir/a1
1601 $LFS mkdir -i 1 $DIR/$tdir/a2
1602 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1603 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1604 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1605 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1607 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1609 $LFS path2fid $DIR/$tdir/a1/f1
1610 $LFS getstripe $DIR/$tdir/a1/f1
1611 $LFS path2fid $DIR/$tdir/a2/f2
1612 $LFS getstripe $DIR/$tdir/a2/f2
1614 cancel_lru_locks osc
1616 echo "Inject failure, to make the MDT-object lost its layout EA"
1617 #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
1618 do_facet mds1 $LCTL set_param fail_loc=0x1615
1619 chown 1.1 $DIR/$tdir/a1/f1
1620 do_facet mds2 $LCTL set_param fail_loc=0x1615
1621 chown 1.1 $DIR/$tdir/a2/f2
1624 do_facet mds1 $LCTL set_param fail_loc=0
1625 do_facet mds2 $LCTL set_param fail_loc=0
1627 echo "stopall to cleanup object cache"
1630 setupall > /dev/null
1632 echo "The file size should be incorrect since layout EA is lost"
1633 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1634 [ "$cur_size" != "$saved_size" ] ||
1635 error "(1) Expect incorrect file1 size"
1637 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1638 [ "$cur_size" != "$saved_size" ] ||
1639 error "(2) Expect incorrect file2 size"
1641 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1642 $START_LAYOUT -o || error "(3) Fail to start LFSCK for layout!"
1644 for k in $(seq $MDSCOUNT); do
1645 # The LFSCK status query internal is 30 seconds. For the case
1646 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1647 # time to guarantee the status sync up.
1648 wait_update_facet mds${k} "$LCTL get_param -n \
1649 mdd.$(facet_svc mds${k}).lfsck_layout |
1650 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1651 error "(4) MDS${k} is not the expected 'completed'"
1654 for k in $(seq $OSTCOUNT); do
1655 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1656 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1657 awk '/^status/ { print $2 }')
1658 [ "$cur_status" == "completed" ] ||
1659 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1663 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1664 mdd.$(facet_svc mds${k}).lfsck_layout |
1665 awk '/^repaired_orphan/ { print $2 }')
1666 [ $repaired -eq ${k} ] ||
1667 error "(6) Expect ${k} fixed on mds${k}, but got: $repaired"
1670 $LFS path2fid $DIR/$tdir/a1/f1
1671 $LFS getstripe $DIR/$tdir/a1/f1
1672 $LFS path2fid $DIR/$tdir/a2/f2
1673 $LFS getstripe $DIR/$tdir/a2/f2
1675 echo "The file size should be correct after layout LFSCK scanning"
1676 cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1677 [ "$cur_size" == "$saved_size" ] ||
1678 error "(7) Expect file1 size $saved_size, but got $cur_size"
1680 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1681 [ "$cur_size" == "$saved_size" ] ||
1682 error "(8) Expect file2 size $saved_size, but got $cur_size"
1684 run_test 18a "Find out orphan OST-object and repair it (1)"
1687 [ $MDSCOUNT -lt 2 ] &&
1688 skip "We need at least 2 MDSes for test_18b" && exit 0
1690 [ $OSTCOUNT -lt 2 ] &&
1691 skip "We need at least 2 OSTs for test_18b" && exit 0
1694 echo "The target MDT-object is lost. The LFSCK should re-create the"
1695 echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
1696 echo "can move it back to normal namespace manually."
1702 formatall > /dev/null
1704 setupall > /dev/null
1707 $LFS mkdir -i 0 $DIR/$tdir/a1
1708 $LFS mkdir -i 1 $DIR/$tdir/a2
1709 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1710 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1711 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1712 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1713 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1714 local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
1716 $LFS getstripe $DIR/$tdir/a1/f1
1717 local fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
1719 $LFS getstripe $DIR/$tdir/a2/f2
1721 cancel_lru_locks osc
1723 echo "Inject failure, to simulate the case of missing the MDT-object"
1724 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1725 do_facet mds1 $LCTL set_param fail_loc=0x1616
1726 rm -f $DIR/$tdir/a1/f1
1727 do_facet mds2 $LCTL set_param fail_loc=0x1616
1728 rm -f $DIR/$tdir/a2/f2
1731 do_facet mds1 $LCTL set_param fail_loc=0
1732 do_facet mds2 $LCTL set_param fail_loc=0
1734 echo "stopall to cleanup object cache"
1737 setupall > /dev/null
1739 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1740 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1742 for k in $(seq $MDSCOUNT); do
1743 # The LFSCK status query internal is 30 seconds. For the case
1744 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1745 # time to guarantee the status sync up.
1746 wait_update_facet mds${k} "$LCTL get_param -n \
1747 mdd.$(facet_svc mds${k}).lfsck_layout |
1748 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1749 error "(2) MDS${k} is not the expected 'completed'"
1752 for k in $(seq $OSTCOUNT); do
1753 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1754 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1755 awk '/^status/ { print $2 }')
1756 [ "$cur_status" == "completed" ] ||
1757 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1761 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1762 mdd.$(facet_svc mds${k}).lfsck_layout |
1763 awk '/^repaired_orphan/ { print $2 }')
1764 [ $repaired -eq ${k} ] ||
1765 error "(4) Expect ${k} fixed on mds${k}, but got: $repaired"
1768 echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
1769 mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
1770 error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
1772 mv $MOUNT/.lustre/lost+found/MDT0001/R-${fid2} $DIR/$tdir/a2/f2 ||
1773 error "(6) Fail to move $MOUNT/.lustre/lost+found/MDT0001/R-${fid2}"
1775 $LFS path2fid $DIR/$tdir/a1/f1
1776 $LFS getstripe $DIR/$tdir/a1/f1
1777 $LFS path2fid $DIR/$tdir/a2/f2
1778 $LFS getstripe $DIR/$tdir/a2/f2
1780 echo "The file size should be correct after layout LFSCK scanning"
1781 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1782 [ "$cur_size" == "$saved_size" ] ||
1783 error "(7) Expect file1 size $saved_size, but got $cur_size"
1785 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1786 [ "$cur_size" == "$saved_size" ] ||
1787 error "(8) Expect file2 size $saved_size, but got $cur_size"
1789 run_test 18b "Find out orphan OST-object and repair it (2)"
1792 [ $MDSCOUNT -lt 2 ] &&
1793 skip "We need at least 2 MDSes for test_18c" && exit 0
1795 [ $OSTCOUNT -lt 2 ] &&
1796 skip "We need at least 2 OSTs for test_18c" && exit 0
1799 echo "The target MDT-object is lost, and the OST-object FID is missing."
1800 echo "The LFSCK should re-create the MDT-object with new FID under the "
1801 echo "directory .lustre/lost+found/MDTxxxx."
1807 formatall > /dev/null
1809 setupall > /dev/null
1812 $LFS mkdir -i 0 $DIR/$tdir/a1
1813 $LFS mkdir -i 1 $DIR/$tdir/a2
1814 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1815 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1817 echo "Inject failure, to simulate the case of missing parent FID"
1818 #define OBD_FAIL_LFSCK_NOPFID 0x1617
1819 do_facet ost1 $LCTL set_param fail_loc=0x1617
1820 do_facet ost2 $LCTL set_param fail_loc=0x1617
1822 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1823 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1824 $LFS getstripe $DIR/$tdir/a1/f1
1825 $LFS getstripe $DIR/$tdir/a2/f2
1827 cancel_lru_locks osc
1829 echo "Inject failure, to simulate the case of missing the MDT-object"
1830 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1831 do_facet mds1 $LCTL set_param fail_loc=0x1616
1832 rm -f $DIR/$tdir/a1/f1
1833 do_facet mds2 $LCTL set_param fail_loc=0x1616
1834 rm -f $DIR/$tdir/a2/f2
1837 do_facet mds1 $LCTL set_param fail_loc=0
1838 do_facet mds2 $LCTL set_param fail_loc=0
1840 echo "stopall to cleanup object cache"
1843 setupall > /dev/null
1845 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1846 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1848 for k in $(seq $MDSCOUNT); do
1849 # The LFSCK status query internal is 30 seconds. For the case
1850 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1851 # time to guarantee the status sync up.
1852 wait_update_facet mds${k} "$LCTL get_param -n \
1853 mdd.$(facet_svc mds${k}).lfsck_layout |
1854 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1855 error "(2) MDS${k} is not the expected 'completed'"
1858 for k in $(seq $OSTCOUNT); do
1859 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1860 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1861 awk '/^status/ { print $2 }')
1862 [ "$cur_status" == "completed" ] ||
1863 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1866 local repaired=$(do_facet mds1 $LCTL get_param -n \
1867 mdd.$(facet_svc mds1).lfsck_layout |
1868 awk '/^repaired_orphan/ { print $2 }')
1869 [ $repaired -eq 3 ] ||
1870 error "(4) Expect 3 fixed on mds1, but got: $repaired"
1872 repaired=$(do_facet mds2 $LCTL get_param -n \
1873 mdd.$(facet_svc mds2).lfsck_layout |
1874 awk '/^repaired_orphan/ { print $2 }')
1875 [ $repaired -eq 0 ] ||
1876 error "(5) Expect 0 fixed on mds2, but got: $repaired"
1878 echo "There should be some stub under .lustre/lost+found/MDT0001/"
1879 ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
1880 error "(6) .lustre/lost+found/MDT0001/ should be empty"
1882 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1883 ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
1884 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1886 run_test 18c "Find out orphan OST-object and repair it (3)"
1890 echo "The target MDT-object layout EA slot is occpuied by some new"
1891 echo "created OST-object when repair dangling reference case. Such"
1892 echo "conflict OST-object has never been modified. Then when found"
1893 echo "the orphan OST-object, LFSCK will replace it with the orphan"
1900 formatall > /dev/null
1902 setupall > /dev/null
1904 mkdir -p $DIR/$tdir/a1
1905 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1906 echo "guard" > $DIR/$tdir/a1/f1
1907 echo "foo" > $DIR/$tdir/a1/f2
1908 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1909 $LFS path2fid $DIR/$tdir/a1/f1
1910 $LFS getstripe $DIR/$tdir/a1/f1
1911 $LFS path2fid $DIR/$tdir/a1/f2
1912 $LFS getstripe $DIR/$tdir/a1/f2
1914 cancel_lru_locks osc
1916 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1917 echo "to reference the same OST-object (which is f1's OST-obejct)."
1918 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1919 echo "dangling reference case, but f2's old OST-object is there."
1922 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1923 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1924 chown 1.1 $DIR/$tdir/a1/f2
1925 rm -f $DIR/$tdir/a1/f1
1928 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1930 echo "stopall to cleanup object cache"
1933 setupall > /dev/null
1935 echo "The file size should be incorrect since dangling referenced"
1936 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1937 [ "$cur_size" != "$saved_size" ] ||
1938 error "(1) Expect incorrect file2 size"
1940 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1941 $START_LAYOUT -o || error "(2) Fail to start LFSCK for layout!"
1943 for k in $(seq $MDSCOUNT); do
1944 # The LFSCK status query internal is 30 seconds. For the case
1945 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1946 # time to guarantee the status sync up.
1947 wait_update_facet mds${k} "$LCTL get_param -n \
1948 mdd.$(facet_svc mds${k}).lfsck_layout |
1949 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1950 error "(3) MDS${k} is not the expected 'completed'"
1953 for k in $(seq $OSTCOUNT); do
1954 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1955 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1956 awk '/^status/ { print $2 }')
1957 [ "$cur_status" == "completed" ] ||
1958 error "(4) OST${k} Expect 'completed', but got '$cur_status'"
1961 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1962 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1963 awk '/^repaired_orphan/ { print $2 }')
1964 [ $repaired -eq 1 ] ||
1965 error "(5) Expect 1 orphan has been fixed, but got: $repaired"
1967 echo "The file size should be correct after layout LFSCK scanning"
1968 cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1969 [ "$cur_size" == "$saved_size" ] ||
1970 error "(6) Expect file2 size $saved_size, but got $cur_size"
1972 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1973 ls -ail $MOUNT/.lustre/lost+found/MDT0000/ &&
1974 error "(7) .lustre/lost+found/MDT0000/ should be empty"
1976 echo "The LFSCK should find back the original data."
1977 cat $DIR/$tdir/a1/f2
1978 $LFS path2fid $DIR/$tdir/a1/f2
1979 $LFS getstripe $DIR/$tdir/a1/f2
1981 run_test 18d "Find out orphan OST-object and repair it (4)"
1985 echo "The target MDT-object layout EA slot is occpuied by some new"
1986 echo "created OST-object when repair dangling reference case. Such"
1987 echo "conflict OST-object has been modified by others. To keep the"
1988 echo "new data, the LFSCK will create a new file to refernece this"
1989 echo "old orphan OST-object."
1995 formatall > /dev/null
1997 setupall > /dev/null
1999 mkdir -p $DIR/$tdir/a1
2000 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
2001 echo "guard" > $DIR/$tdir/a1/f1
2002 echo "foo" > $DIR/$tdir/a1/f2
2003 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2004 $LFS path2fid $DIR/$tdir/a1/f1
2005 $LFS getstripe $DIR/$tdir/a1/f1
2006 $LFS path2fid $DIR/$tdir/a1/f2
2007 $LFS getstripe $DIR/$tdir/a1/f2
2009 cancel_lru_locks osc
2011 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
2012 echo "to reference the same OST-object (which is f1's OST-obejct)."
2013 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
2014 echo "dangling reference case, but f2's old OST-object is there."
2017 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
2018 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
2019 chown 1.1 $DIR/$tdir/a1/f2
2020 rm -f $DIR/$tdir/a1/f1
2023 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2025 echo "stopall to cleanup object cache"
2028 setupall > /dev/null
2030 echo "The file size should be incorrect since dangling referenced"
2031 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2032 [ "$cur_size" != "$saved_size" ] ||
2033 error "(1) Expect incorrect file2 size"
2035 #define OBD_FAIL_LFSCK_DELAY3 0x1602
2036 do_facet $SINGLEMDS $LCTL set_param fail_val=10
2037 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
2039 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
2040 $START_LAYOUT -o || error "(2) Fail to start LFSCK for layout!"
2042 wait_update_facet mds1 "$LCTL get_param -n \
2043 mdd.$(facet_svc mds1).lfsck_layout |
2044 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
2045 error "(3) MDS1 is not the expected 'scanning-phase2'"
2047 # to guarantee all updates are synced.
2051 echo "Write new data to f2 to modify the new created OST-object."
2052 echo "dummy" >> $DIR/$tdir/a1/f2
2054 do_facet $SINGLEMDS $LCTL set_param fail_val=0
2055 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2057 for k in $(seq $MDSCOUNT); do
2058 # The LFSCK status query internal is 30 seconds. For the case
2059 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
2060 # time to guarantee the status sync up.
2061 wait_update_facet mds${k} "$LCTL get_param -n \
2062 mdd.$(facet_svc mds${k}).lfsck_layout |
2063 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
2064 error "(4) MDS${k} is not the expected 'completed'"
2067 for k in $(seq $OSTCOUNT); do
2068 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
2069 obdfilter.$(facet_svc ost${k}).lfsck_layout |
2070 awk '/^status/ { print $2 }')
2071 [ "$cur_status" == "completed" ] ||
2072 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
2075 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
2076 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
2077 awk '/^repaired_orphan/ { print $2 }')
2078 [ $repaired -eq 1 ] ||
2079 error "(6) Expect 1 orphan has been fixed, but got: $repaired"
2081 echo "There should be stub file under .lustre/lost+found/MDT0000/"
2082 local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
2084 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
2086 echo "The stub file should keep the original f2 data"
2087 cur_size=$(ls -il $cname | awk '{ print $6 }')
2088 [ "$cur_size" == "$saved_size" ] ||
2089 error "(8) Expect file2 size $saved_size, but got $cur_size"
2092 $LFS path2fid $cname
2093 $LFS getstripe $cname
2095 echo "The f2 should contains new data."
2096 cat $DIR/$tdir/a1/f2
2097 $LFS path2fid $DIR/$tdir/a1/f2
2098 $LFS getstripe $DIR/$tdir/a1/f2
2100 run_test 18e "Find out orphan OST-object and repair it (5)"
2106 formatall > /dev/null
2108 setupall > /dev/null
2111 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2113 echo "foo" > $DIR/$tdir/a0
2114 echo "guard" > $DIR/$tdir/a1
2116 cancel_lru_locks osc
2117 umount_client $MOUNT || error "(1) Fail to stop client!"
2118 mount_client $MOUNT || error "(2) Fail to start client!"
2120 echo "Inject failure, then client will offer wrong parent FID when read"
2121 do_facet ost1 $LCTL set_param -n \
2122 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2123 #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
2124 $LCTL set_param fail_loc=0x1619
2126 echo "Read RPC with wrong parent FID should be denied"
2127 cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
2128 $LCTL set_param fail_loc=0
2130 run_test 19a "OST-object inconsistency self detect"
2136 formatall > /dev/null
2138 setupall > /dev/null
2141 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2143 echo "Inject failure stub to make the OST-object to back point to"
2144 echo "non-exist MDT-object"
2146 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
2147 do_facet ost1 $LCTL set_param fail_loc=0x1611
2148 echo "foo" > $DIR/$tdir/f0
2149 cancel_lru_locks osc
2152 do_facet ost1 $LCTL set_param fail_loc=0
2154 echo "Nothing should be fixed since self detect and repair is disabled"
2155 local repaired=$(do_facet ost1 $LCTL get_param -n \
2156 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2157 awk '/^repaired/ { print $2 }')
2158 [ $repaired -eq 0 ] ||
2159 error "(1) Expected 0 repaired, but got $repaired"
2161 echo "Read RPC with right parent FID should be accepted,"
2162 echo "and cause parent FID on OST to be fixed"
2164 do_facet ost1 $LCTL set_param -n \
2165 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2166 cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
2168 repaired=$(do_facet ost1 $LCTL get_param -n \
2169 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2170 awk '/^repaired/ { print $2 }')
2171 [ $repaired -eq 1 ] ||
2172 error "(3) Expected 1 repaired, but got $repaired"
2174 run_test 19b "OST-object inconsistency self repair"
2176 $LCTL set_param debug=-lfsck > /dev/null || true
2178 # restore MDS/OST size
2179 MDSSIZE=${SAVED_MDSSIZE}
2180 OSTSIZE=${SAVED_OSTSIZE}
2182 # cleanup the system at last