3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 MCREATE=${MCREATE:-mcreate}
23 SAVED_MDSSIZE=${MDSSIZE}
24 SAVED_OSTSIZE=${OSTSIZE}
25 # use small MDS + OST size to speed formatting time
26 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
33 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
36 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
37 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
39 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
44 $LCTL set_param debug=+lfsck > /dev/null || true
46 MDT_DEV="${FSNAME}-MDT0000"
47 OST_DEV="${FSNAME}-OST0000"
48 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
49 START_NAMESPACE="do_facet $SINGLEMDS \
50 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
51 START_LAYOUT="do_facet $SINGLEMDS \
52 $LCTL lfsck_start -M ${MDT_DEV} -t layout"
53 START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
54 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
55 SHOW_NAMESPACE="do_facet $SINGLEMDS \
56 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
57 SHOW_LAYOUT="do_facet $SINGLEMDS \
58 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
59 SHOW_LAYOUT_ON_OST="do_facet ost1 \
60 $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
61 MOUNT_OPTS_SCRUB="-o user_xattr"
62 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
75 if [ ! -z $igif ]; then
76 #define OBD_FAIL_FID_IGIF 0x1504
77 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
80 echo "preparing... ${nfiles} * ${ndirs} files will be created."
82 cp $LUSTRE/tests/*.sh $DIR/
83 for ((i = 0; i < ${ndirs}; i++)); do
84 mkdir $DIR/$tdir/d${i}
85 touch $DIR/$tdir/f${i}
86 for ((j = 0; j < ${nfiles}; j++)); do
87 touch $DIR/$tdir/d${i}/f${j}
89 mkdir $DIR/$tdir/e${i}
92 if [ ! -z $igif ]; then
93 touch $DIR/$tdir/dummy
94 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
98 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
99 echo "stop $SINGLEMDS"
100 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
105 echo "start $SINGLEMDS"
106 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
107 error "(1) Fail to start MDS!"
109 #define OBD_FAIL_LFSCK_DELAY1 0x1600
110 do_facet $SINGLEMDS $LCTL set_param fail_val=3
111 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
112 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
114 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
116 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
117 [ "$STATUS" == "scanning-phase1" ] ||
118 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
120 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
122 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
123 [ "$STATUS" == "stopped" ] ||
124 error "(6) Expect 'stopped', but got '$STATUS'"
126 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
128 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
129 [ "$STATUS" == "scanning-phase1" ] ||
130 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
132 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
133 do_facet $SINGLEMDS $LCTL set_param fail_val=0
134 wait_update_facet $SINGLEMDS \
135 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
136 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
137 error "(9) unexpected status"
139 local repaired=$($SHOW_NAMESPACE |
140 awk '/^updated_phase1/ { print $2 }')
141 [ $repaired -eq 0 ] ||
142 error "(10) Expect nothing to be repaired, but got: $repaired"
144 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
145 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
146 wait_update_facet $SINGLEMDS \
147 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
148 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
149 error "(12) unexpected status"
151 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
152 [ $((scanned1 + 1)) -eq $scanned2 ] ||
153 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
155 echo "stopall, should NOT crash LU-3649"
158 run_test 0 "Control LFSCK manually"
161 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
162 skip "OI Scrub not implemented for ZFS" && return
165 echo "start $SINGLEMDS"
166 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
167 error "(1) Fail to start MDS!"
169 mount_client $MOUNT || error "(2) Fail to start client!"
171 #define OBD_FAIL_FID_INDIR 0x1501
172 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
173 touch $DIR/$tdir/dummy
175 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
177 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
180 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
181 [ "$STATUS" == "completed" ] ||
182 error "(4) Expect 'completed', but got '$STATUS'"
184 local repaired=$($SHOW_NAMESPACE |
185 awk '/^updated_phase1/ { print $2 }')
186 [ $repaired -eq 1 ] ||
187 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
189 mount_client $MOUNT || error "(6) Fail to start client!"
191 #define OBD_FAIL_FID_LOOKUP 0x1505
192 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
193 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
195 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
197 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
201 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
202 skip "OI Scrub not implemented for ZFS" && return
205 echo "start $SINGLEMDS"
206 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
207 error "(1) Fail to start MDS!"
209 mount_client $MOUNT || error "(2) Fail to start client!"
211 #define OBD_FAIL_FID_INLMA 0x1502
212 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
213 touch $DIR/$tdir/dummy
215 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
217 #define OBD_FAIL_FID_NOLMA 0x1506
218 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
219 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
222 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
223 [ "$STATUS" == "completed" ] ||
224 error "(4) Expect 'completed', but got '$STATUS'"
226 local repaired=$($SHOW_NAMESPACE |
227 awk '/^updated_phase1/ { print $2 }')
228 [ $repaired -eq 1 ] ||
229 error "(5) Fail to repair missed FID-in-LMA: $repaired"
231 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
232 mount_client $MOUNT || error "(6) Fail to start client!"
234 #define OBD_FAIL_FID_LOOKUP 0x1505
235 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
236 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
238 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
240 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
244 echo "start $SINGLEMDS"
245 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
246 error "(1) Fail to start MDS!"
248 mount_client $MOUNT || error "(2) Fail to start client!"
250 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
251 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
252 touch $DIR/$tdir/dummy
254 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
256 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
259 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
260 [ "$STATUS" == "completed" ] ||
261 error "(4) Expect 'completed', but got '$STATUS'"
263 local repaired=$($SHOW_NAMESPACE |
264 awk '/^updated_phase1/ { print $2 }')
265 [ $repaired -eq 1 ] ||
266 error "(5) Fail to repair crashed linkEA: $repaired"
268 mount_client $MOUNT || error "(6) Fail to start client!"
270 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
271 error "(7) Fail to stat $DIR/$tdir/dummy"
273 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
274 local dummyname=$($LFS fid2path $DIR $dummyfid)
275 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
276 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
278 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
283 echo "start $SINGLEMDS"
284 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
285 error "(1) Fail to start MDS!"
287 mount_client $MOUNT || error "(2) Fail to start client!"
289 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
290 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
291 touch $DIR/$tdir/dummy
293 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
295 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
298 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
299 [ "$STATUS" == "completed" ] ||
300 error "(4) Expect 'completed', but got '$STATUS'"
302 local repaired=$($SHOW_NAMESPACE |
303 awk '/^updated_phase2/ { print $2 }')
304 [ $repaired -eq 1 ] ||
305 error "(5) Fail to repair crashed linkEA: $repaired"
307 mount_client $MOUNT || error "(6) Fail to start client!"
309 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
310 error "(7) Fail to stat $DIR/$tdir/dummy"
312 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
313 local dummyname=$($LFS fid2path $DIR $dummyfid)
314 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
315 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
317 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
322 echo "start $SINGLEMDS"
323 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
324 error "(1) Fail to start MDS!"
326 mount_client $MOUNT || error "(2) Fail to start client!"
328 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
330 touch $DIR/$tdir/dummy
332 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
334 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
337 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
338 [ "$STATUS" == "completed" ] ||
339 error "(4) Expect 'completed', but got '$STATUS'"
341 local repaired=$($SHOW_NAMESPACE |
342 awk '/^updated_phase2/ { print $2 }')
343 [ $repaired -eq 1 ] ||
344 error "(5) Fail to repair crashed linkEA: $repaired"
346 mount_client $MOUNT || error "(6) Fail to start client!"
348 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
349 error "(7) Fail to stat $DIR/$tdir/dummy"
351 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
352 local dummyname=$($LFS fid2path $DIR $dummyfid)
353 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
354 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
356 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
360 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
361 skip "OI Scrub not implemented for ZFS" && return
364 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
365 echo "start $SINGLEMDS with disabling OI scrub"
366 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
367 error "(2) Fail to start MDS!"
369 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
370 [ "$STATUS" == "init" ] ||
371 error "(3) Expect 'init', but got '$STATUS'"
373 #define OBD_FAIL_LFSCK_DELAY2 0x1601
374 do_facet $SINGLEMDS $LCTL set_param fail_val=1
375 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
376 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
379 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
380 [ "$STATUS" == "scanning-phase1" ] ||
381 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
383 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
384 [ "$FLAGS" == "inconsistent" ] ||
385 error "(6) Expect 'inconsistent', but got '$FLAGS'"
387 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
388 do_facet $SINGLEMDS $LCTL set_param fail_val=0
390 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
391 [ "$STATUS" == "completed" ] ||
392 error "(7) Expect 'completed', but got '$STATUS'"
394 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
395 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
397 local repaired=$($SHOW_NAMESPACE |
398 awk '/^updated_phase1/ { print $2 }')
399 [ $repaired -ge 9 ] ||
400 error "(9) Fail to repair crashed linkEA: $repaired"
402 mount_client $MOUNT || error "(10) Fail to start client!"
404 #define OBD_FAIL_FID_LOOKUP 0x1505
405 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
406 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
408 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
410 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
414 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
415 skip "OI Scrub not implemented for ZFS" && return
418 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
419 echo "start $SINGLEMDS with disabling OI scrub"
420 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
421 error "(2) Fail to start MDS!"
423 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
424 [ "$STATUS" == "init" ] ||
425 error "(3) Expect 'init', but got '$STATUS'"
427 #define OBD_FAIL_LFSCK_DELAY2 0x1601
428 do_facet $SINGLEMDS $LCTL set_param fail_val=1
429 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
430 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
433 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
434 [ "$STATUS" == "scanning-phase1" ] ||
435 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
437 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
438 [ "$FLAGS" == "inconsistent,upgrade" ] ||
439 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
441 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
442 do_facet $SINGLEMDS $LCTL set_param fail_val=0
444 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
445 [ "$STATUS" == "completed" ] ||
446 error "(7) Expect 'completed', but got '$STATUS'"
448 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
449 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
451 local repaired=$($SHOW_NAMESPACE |
452 awk '/^updated_phase1/ { print $2 }')
453 [ $repaired -ge 2 ] ||
454 error "(9) Fail to repair crashed linkEA: $repaired"
456 mount_client $MOUNT || error "(10) Fail to start client!"
458 #define OBD_FAIL_FID_LOOKUP 0x1505
459 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
460 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
462 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
464 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
465 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
466 local dummyname=$($LFS fid2path $DIR $dummyfid)
467 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
468 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
470 run_test 5 "LFSCK can handle IFIG object upgrading"
474 echo "start $SINGLEMDS"
475 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
476 error "(1) Fail to start MDS!"
478 #define OBD_FAIL_LFSCK_DELAY1 0x1600
479 do_facet $SINGLEMDS $LCTL set_param fail_val=1
480 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
481 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
483 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
484 [ "$STATUS" == "scanning-phase1" ] ||
485 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
487 # Sleep 3 sec to guarantee at least one object processed by LFSCK
489 # Fail the LFSCK to guarantee there is at least one checkpoint
490 #define OBD_FAIL_LFSCK_FATAL1 0x1608
491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
493 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
494 [ "$STATUS" == "failed" ] ||
495 error "(4) Expect 'failed', but got '$STATUS'"
497 local POSITION0=$($SHOW_NAMESPACE |
498 awk '/^last_checkpoint_position/ { print $2 }' |
501 #define OBD_FAIL_LFSCK_DELAY1 0x1600
502 do_facet $SINGLEMDS $LCTL set_param fail_val=1
503 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
504 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
506 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
507 [ "$STATUS" == "scanning-phase1" ] ||
508 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
510 local POSITION1=$($SHOW_NAMESPACE |
511 awk '/^latest_start_position/ { print $2 }' |
513 [ $POSITION0 -lt $POSITION1 ] ||
514 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
516 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
517 do_facet $SINGLEMDS $LCTL set_param fail_val=0
519 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
520 [ "$STATUS" == "completed" ] ||
521 error "(8) Expect 'completed', but got '$STATUS'"
523 run_test 6a "LFSCK resumes from last checkpoint (1)"
527 echo "start $SINGLEMDS"
528 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
529 error "(1) Fail to start MDS!"
531 #define OBD_FAIL_LFSCK_DELAY2 0x1601
532 do_facet $SINGLEMDS $LCTL set_param fail_val=1
533 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
534 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
536 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
537 [ "$STATUS" == "scanning-phase1" ] ||
538 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
540 # Sleep 3 sec to guarantee at least one object processed by LFSCK
542 # Fail the LFSCK to guarantee there is at least one checkpoint
543 #define OBD_FAIL_LFSCK_FATAL2 0x1609
544 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
546 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
547 [ "$STATUS" == "failed" ] ||
548 error "(4) Expect 'failed', but got '$STATUS'"
550 local POSITION0=$($SHOW_NAMESPACE |
551 awk '/^last_checkpoint_position/ { print $4 }')
553 #define OBD_FAIL_LFSCK_DELAY2 0x1601
554 do_facet $SINGLEMDS $LCTL set_param fail_val=1
555 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
556 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
558 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
559 [ "$STATUS" == "scanning-phase1" ] ||
560 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
562 local POSITION1=$($SHOW_NAMESPACE |
563 awk '/^latest_start_position/ { print $4 }')
564 if [ $POSITION0 -gt $POSITION1 ]; then
565 [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
566 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
569 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
570 do_facet $SINGLEMDS $LCTL set_param fail_val=0
572 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "completed" ] ||
574 error "(8) Expect 'completed', but got '$STATUS'"
576 run_test 6b "LFSCK resumes from last checkpoint (2)"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(1) Fail to start MDS!"
585 #define OBD_FAIL_LFSCK_DELAY2 0x1601
586 do_facet $SINGLEMDS $LCTL set_param fail_val=1
587 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
588 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
590 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
591 [ "$STATUS" == "scanning-phase1" ] ||
592 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
594 # Sleep 3 sec to guarantee at least one object processed by LFSCK
596 echo "stop $SINGLEMDS"
597 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
599 echo "start $SINGLEMDS"
600 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
601 error "(5) Fail to start MDS!"
603 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
604 [ "$STATUS" == "scanning-phase1" ] ||
605 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
607 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
608 do_facet $SINGLEMDS $LCTL set_param fail_val=0
610 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
611 [ "$STATUS" == "completed" ] ||
612 error "(7) Expect 'completed', but got '$STATUS'"
614 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
619 echo "start $SINGLEMDS"
620 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
621 error "(1) Fail to start MDS!"
623 mount_client $MOUNT || error "(2) Fail to start client!"
625 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
626 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
627 for ((i = 0; i < 20; i++)); do
628 touch $DIR/$tdir/dummy${i}
631 #define OBD_FAIL_LFSCK_DELAY3 0x1602
632 do_facet $SINGLEMDS $LCTL set_param fail_val=1
633 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
634 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
637 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
638 [ "$STATUS" == "scanning-phase2" ] ||
639 error "(4) Expect 'scanning-phase2', but got '$STATUS'"
641 echo "stop $SINGLEMDS"
642 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
644 echo "start $SINGLEMDS"
645 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
646 error "(6) Fail to start MDS!"
648 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "scanning-phase2" ] ||
650 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
652 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
653 do_facet $SINGLEMDS $LCTL set_param fail_val=0
655 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
656 [ "$STATUS" == "completed" ] ||
657 error "(8) Expect 'completed', but got '$STATUS'"
659 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
664 echo "start $SINGLEMDS"
665 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
666 error "(1) Fail to start MDS!"
668 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
669 [ "$STATUS" == "init" ] ||
670 error "(2) Expect 'init', but got '$STATUS'"
672 mount_client $MOUNT || error "(3) Fail to start client!"
674 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
675 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
676 mkdir $DIR/$tdir/crashed
678 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
679 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
680 for ((i = 0; i < 5; i++)); do
681 touch $DIR/$tdir/dummy${i}
684 #define OBD_FAIL_LFSCK_DELAY2 0x1601
685 do_facet $SINGLEMDS $LCTL set_param fail_val=2
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
687 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
689 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
690 [ "$STATUS" == "scanning-phase1" ] ||
691 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
693 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
695 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
696 [ "$STATUS" == "stopped" ] ||
697 error "(7) Expect 'stopped', but got '$STATUS'"
699 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
701 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
702 [ "$STATUS" == "scanning-phase1" ] ||
703 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
705 #define OBD_FAIL_LFSCK_FATAL2 0x1609
706 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
708 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
709 [ "$STATUS" == "failed" ] ||
710 error "(10) Expect 'failed', but got '$STATUS'"
712 #define OBD_FAIL_LFSCK_DELAY1 0x1600
713 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
714 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
716 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "scanning-phase1" ] ||
718 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
720 #define OBD_FAIL_LFSCK_CRASH 0x160a
721 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
724 echo "stop $SINGLEMDS"
725 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
727 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
728 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
730 echo "start $SINGLEMDS"
731 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
732 error "(14) Fail to start MDS!"
734 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
735 [ "$STATUS" == "crashed" ] ||
736 error "(15) Expect 'crashed', but got '$STATUS'"
738 #define OBD_FAIL_LFSCK_DELAY2 0x1601
739 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
740 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
742 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
743 [ "$STATUS" == "scanning-phase1" ] ||
744 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
746 echo "stop $SINGLEMDS"
747 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
749 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
750 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
752 echo "start $SINGLEMDS"
753 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
754 error "(19) Fail to start MDS!"
756 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
757 [ "$STATUS" == "paused" ] ||
758 error "(20) Expect 'paused', but got '$STATUS'"
760 #define OBD_FAIL_LFSCK_DELAY3 0x1602
761 do_facet $SINGLEMDS $LCTL set_param fail_val=2
762 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
764 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
766 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
767 [ "$STATUS" == "scanning-phase2" ] ||
768 error "(22) Expect 'scanning-phase2', but got '$STATUS'"
770 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
771 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
772 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
774 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
775 do_facet $SINGLEMDS $LCTL set_param fail_val=0
777 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
778 [ "$STATUS" == "completed" ] ||
779 error "(24) Expect 'completed', but got '$STATUS'"
781 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
782 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
785 run_test 8 "LFSCK state machine"
788 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
789 skip "Testing on UP system, the speed may be inaccurate."
794 echo "start $SINGLEMDS"
795 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
796 error "(1) Fail to start MDS!"
798 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
799 [ "$STATUS" == "init" ] ||
800 error "(2) Expect 'init', but got '$STATUS'"
802 local BASE_SPEED1=100
804 $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
807 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
808 [ "$STATUS" == "scanning-phase1" ] ||
809 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
811 local SPEED=$($SHOW_NAMESPACE |
812 awk '/^average_speed_phase1/ { print $2 }')
814 # There may be time error, normally it should be less than 2 seconds.
815 # We allow another 20% schedule error.
817 # MAX_MARGIN = 1.2 = 12 / 10
818 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
819 RUN_TIME1 * 12 / 10))
820 [ $SPEED -lt $MAX_SPEED ] ||
821 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
824 local BASE_SPEED2=300
826 do_facet $SINGLEMDS \
827 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
830 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
831 # MIN_MARGIN = 0.8 = 8 / 10
832 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
833 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
834 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
835 [ $SPEED -gt $MIN_SPEED ] ||
836 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
838 # MAX_MARGIN = 1.2 = 12 / 10
839 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
840 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
841 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
842 [ $SPEED -lt $MAX_SPEED ] ||
843 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
845 do_facet $SINGLEMDS \
846 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
848 wait_update_facet $SINGLEMDS \
849 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
850 awk '/^status/ { print \\\$2 }'" "completed" 30 ||
851 error "(7) Failed to get expected 'completed'"
853 run_test 9a "LFSCK speed control (1)"
856 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
857 skip "Testing on UP system, the speed may be inaccurate."
862 echo "start $SINGLEMDS"
863 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
864 error "(1) Fail to start MDS!"
866 mount_client $MOUNT || error "(2) Fail to start client!"
868 echo "Another preparing... 50 * 50 files (with error) will be created."
869 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
870 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
871 for ((i = 0; i < 50; i++)); do
872 mkdir -p $DIR/$tdir/d${i}
873 touch $DIR/$tdir/f${i}
874 for ((j = 0; j < 50; j++)); do
875 touch $DIR/$tdir/d${i}/f${j}
879 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
880 [ "$STATUS" == "init" ] ||
881 error "(3) Expect 'init', but got '$STATUS'"
883 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
884 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
885 $START_NAMESPACE || error "(4) Fail to start LFSCK!"
888 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
889 [ "$STATUS" == "stopped" ] ||
890 error "(5) Expect 'stopped', but got '$STATUS'"
892 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
896 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
899 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
900 [ "$STATUS" == "scanning-phase2" ] ||
901 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
903 local SPEED=$($SHOW_NAMESPACE |
904 awk '/^average_speed_phase2/ { print $2 }')
905 # There may be time error, normally it should be less than 2 seconds.
906 # We allow another 20% schedule error.
908 # MAX_MARGIN = 1.2 = 12 / 10
909 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
910 RUN_TIME1 * 12 / 10))
911 [ $SPEED -lt $MAX_SPEED ] ||
912 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
915 local BASE_SPEED2=150
917 do_facet $SINGLEMDS \
918 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
921 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
922 # MIN_MARGIN = 0.8 = 8 / 10
923 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
924 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
925 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
926 [ $SPEED -gt $MIN_SPEED ] ||
927 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
929 # MAX_MARGIN = 1.2 = 12 / 10
930 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
931 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
932 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
933 [ $SPEED -lt $MAX_SPEED ] ||
934 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
936 do_facet $SINGLEMDS \
937 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
939 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
940 [ "$STATUS" == "completed" ] ||
941 error "(11) Expect 'completed', but got '$STATUS'"
943 run_test 9b "LFSCK speed control (2)"
947 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
948 skip "lookup(..)/linkea on ZFS issue" && return
951 echo "start $SINGLEMDS"
952 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
953 error "(1) Fail to start MDS!"
955 mount_client $MOUNT || error "(2) Fail to start client!"
957 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
958 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
959 for ((i = 0; i < 1000; i = $((i+2)))); do
960 mkdir -p $DIR/$tdir/d${i}
961 touch $DIR/$tdir/f${i}
962 for ((j = 0; j < 5; j++)); do
963 touch $DIR/$tdir/d${i}/f${j}
967 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
968 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
969 for ((i = 1; i < 1000; i = $((i+2)))); do
970 mkdir -p $DIR/$tdir/d${i}
971 touch $DIR/$tdir/f${i}
972 for ((j = 0; j < 5; j++)); do
973 touch $DIR/$tdir/d${i}/f${j}
977 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
978 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
981 mount_client $MOUNT || error "(3) Fail to start client!"
983 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
984 [ "$STATUS" == "init" ] ||
985 error "(4) Expect 'init', but got '$STATUS'"
987 $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
990 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
991 [ "$STATUS" == "scanning-phase1" ] ||
992 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
994 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
996 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
998 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
1000 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
1002 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
1004 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
1006 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
1008 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
1009 error "(14) Fail to softlink!"
1011 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1012 [ "$STATUS" == "scanning-phase1" ] ||
1013 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
1015 do_facet $SINGLEMDS \
1016 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
1017 umount_client $MOUNT
1019 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1020 [ "$STATUS" == "completed" ] ||
1021 error "(16) Expect 'completed', but got '$STATUS'"
1023 run_test 10 "System is available during LFSCK scanning"
1026 ost_remove_lastid() {
1029 local rcmd="do_facet ost${ost}"
1031 echo "remove LAST_ID on ost${ost}: idx=${idx}"
1033 # step 1: local mount
1034 mount_fstype ost${ost} || return 1
1035 # step 2: remove the specified LAST_ID
1036 ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/{LAST_ID,d0/0}
1038 unmount_fstype ost${ost} || return 2
1045 formatall > /dev/null
1047 setupall > /dev/null
1050 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1051 createmany -o $DIR/$tdir/f 64
1056 ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
1059 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
1060 error "(2) Fail to start ost1"
1062 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1063 [ "$STATUS" == "init" ] ||
1064 error "(3) Expect 'init', but got '$STATUS'"
1066 #define OBD_FAIL_LFSCK_DELAY4 0x160e
1067 do_facet ost1 $LCTL set_param fail_val=3
1068 do_facet ost1 $LCTL set_param fail_loc=0x160e
1070 echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
1071 $START_LAYOUT_ON_OST || error "(4) Fail to start LFSCK on OST!"
1073 wait_update_facet ost1 "$LCTL get_param -n \
1074 obdfilter.${OST_DEV}.lfsck_layout |
1075 awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
1080 do_facet ost1 $LCTL set_param fail_val=0
1081 do_facet ost1 $LCTL set_param fail_loc=0
1083 wait_update_facet ost1 "$LCTL get_param -n \
1084 obdfilter.${OST_DEV}.lfsck_layout |
1085 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1090 echo "the LAST_ID(s) should have been rebuilt"
1091 FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
1092 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
1094 run_test 11a "LFSCK can rebuild lost last_id"
1100 formatall > /dev/null
1102 setupall > /dev/null
1105 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1107 echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
1108 #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
1109 do_facet ost1 $LCTL set_param fail_loc=0x160d
1110 createmany -o $DIR/$tdir/f 64
1111 local lastid1=$(do_facet ost1 "lctl get_param -n \
1112 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1113 awk -F: '{ print $2 }')
1115 umount_client $MOUNT
1117 stop ost1 || error "(1) Fail to stop ost1"
1119 #define OBD_FAIL_OST_ENOSPC 0x215
1120 do_facet ost1 $LCTL set_param fail_loc=0x215
1123 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1124 error "(2) Fail to start ost1"
1126 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1127 [ "$STATUS" == "init" ] ||
1128 error "(3) Expect 'init', but got '$STATUS'"
1130 for ((i = 0; i < 60; i++)); do
1131 lastid2=$(do_facet ost1 "lctl get_param -n \
1132 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1133 awk -F: '{ print $2 }')
1134 [ ! -z $lastid2 ] && break;
1138 echo "the on-disk LAST_ID should be smaller than the expected one"
1139 [ $lastid1 -gt $lastid2 ] ||
1140 error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
1142 echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
1143 $START_LAYOUT_ON_OST || error "(5) Fail to start LFSCK on OST!"
1145 wait_update_facet ost1 "$LCTL get_param -n \
1146 obdfilter.${OST_DEV}.lfsck_layout |
1147 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1153 stop ost1 || error "(7) Fail to stop ost1"
1156 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1157 error "(8) Fail to start ost1"
1159 echo "the on-disk LAST_ID should have been rebuilt"
1160 wait_update_facet ost1 "$LCTL get_param -n \
1161 obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
1162 awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
1163 $LCTL get_param -n obdfilter.${ost1_svc}.last_id
1164 error "(9) expect lastid1 0x100000000:$lastid1"
1167 do_facet ost1 $LCTL set_param fail_loc=0
1169 run_test 11b "LFSCK can rebuild crashed last_id"
1172 [ $MDSCOUNT -lt 2 ] &&
1173 skip "We need at least 2 MDSes for test_12" && exit 0
1178 formatall > /dev/null
1180 setupall > /dev/null
1184 echo "All the LFSCK targets should be in 'init' status."
1185 for k in $(seq $MDSCOUNT); do
1186 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1187 mdd.$(facet_svc mds${k}).lfsck_layout |
1188 awk '/^status/ { print $2 }')
1189 [ "$STATUS" == "init" ] ||
1190 error "(1) MDS${k} Expect 'init', but got '$STATUS'"
1192 $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
1193 createmany -o $DIR/$tdir/${k}/f 100
1196 echo "Start namespace LFSCK on all targets by single command (-s 1)."
1197 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1198 -s 1 || error "(2) Fail to start LFSCK on all devices!"
1200 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1201 for k in $(seq $MDSCOUNT); do
1202 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1203 mdd.$(facet_svc mds${k}).lfsck_namespace |
1204 awk '/^status/ { print $2 }')
1205 [ "$STATUS" == "scanning-phase1" ] ||
1206 error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1209 echo "Stop namespace LFSCK on all targets by single lctl command."
1210 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1211 error "(4) Fail to stop LFSCK on all devices!"
1213 echo "All the LFSCK targets should be in 'stopped' status."
1214 for k in $(seq $MDSCOUNT); do
1215 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1216 mdd.$(facet_svc mds${k}).lfsck_namespace |
1217 awk '/^status/ { print $2 }')
1218 [ "$STATUS" == "stopped" ] ||
1219 error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
1222 echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
1223 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1224 -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
1226 echo "All the LFSCK targets should be in 'completed' status."
1227 for k in $(seq $MDSCOUNT); do
1228 wait_update_facet mds${k} "$LCTL get_param -n \
1229 mdd.$(facet_svc mds${k}).lfsck_namespace |
1230 awk '/^status/ { print \\\$2 }'" "completed" 8 ||
1231 error "(7) MDS${k} is not the expected 'completed'"
1234 echo "Start layout LFSCK on all targets by single command (-s 1)."
1235 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1236 -s 1 || error "(8) Fail to start LFSCK on all devices!"
1238 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1239 for k in $(seq $MDSCOUNT); do
1240 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1241 mdd.$(facet_svc mds${k}).lfsck_layout |
1242 awk '/^status/ { print $2 }')
1243 [ "$STATUS" == "scanning-phase1" ] ||
1244 error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1247 echo "Stop layout LFSCK on all targets by single lctl command."
1248 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1249 error "(10) Fail to stop LFSCK on all devices!"
1251 echo "All the LFSCK targets should be in 'stopped' status."
1252 for k in $(seq $MDSCOUNT); do
1253 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1254 mdd.$(facet_svc mds${k}).lfsck_layout |
1255 awk '/^status/ { print $2 }')
1256 [ "$STATUS" == "stopped" ] ||
1257 error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
1260 for k in $(seq $OSTCOUNT); do
1261 local STATUS=$(do_facet ost${k} $LCTL get_param -n \
1262 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1263 awk '/^status/ { print $2 }')
1264 [ "$STATUS" == "stopped" ] ||
1265 error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
1268 echo "Re-start layout LFSCK on all targets by single command (-s 0)."
1269 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1270 -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
1272 echo "All the LFSCK targets should be in 'completed' status."
1273 for k in $(seq $MDSCOUNT); do
1274 # The LFSCK status query internal is 30 seconds. For the case
1275 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1276 # time to guarantee the status sync up.
1277 wait_update_facet mds${k} "$LCTL get_param -n \
1278 mdd.$(facet_svc mds${k}).lfsck_layout |
1279 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1280 error "(14) MDS${k} is not the expected 'completed'"
1283 run_test 12 "single command to trigger LFSCK on all devices"
1287 echo "The lmm_oi in layout EA should be consistent with the MDT-object"
1288 echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
1289 echo "MDT-object FID."
1295 formatall > /dev/null
1297 setupall > /dev/null
1301 echo "Inject failure stub to simulate bad lmm_oi"
1302 #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
1303 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
1304 createmany -o $DIR/$tdir/f 32
1305 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1307 echo "stopall to cleanup object cache"
1310 setupall > /dev/null
1312 echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
1313 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1315 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1316 mdd.${MDT_DEV}.lfsck_layout |
1317 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1319 local repaired=$($SHOW_LAYOUT |
1320 awk '/^repaired_others/ { print $2 }')
1321 [ $repaired -eq 32 ] ||
1322 error "(3) Fail to repair crashed lmm_oi: $repaired"
1324 run_test 13 "LFSCK can repair crashed lmm_oi"
1328 echo "The OST-object referenced by the MDT-object should be there;"
1329 echo "otherwise, the LFSCK should re-create the missed OST-object."
1335 formatall > /dev/null
1337 setupall > /dev/null
1340 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1342 echo "Inject failure stub to simulate dangling referenced MDT-object"
1343 #define OBD_FAIL_LFSCK_DANGLING 0x1610
1344 do_facet ost1 $LCTL set_param fail_loc=0x1610
1345 createmany -o $DIR/$tdir/f 64
1346 do_facet ost1 $LCTL set_param fail_loc=0
1348 echo "stopall to cleanup object cache"
1351 setupall > /dev/null
1353 echo "'ls' should fail because of dangling referenced MDT-object"
1354 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
1356 echo "Trigger layout LFSCK to find out dangling reference"
1357 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1359 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1360 mdd.${MDT_DEV}.lfsck_layout |
1361 awk '/^status/ { print \\\$2 }'" "completed" 6 || return 3
1363 local repaired=$($SHOW_LAYOUT |
1364 awk '/^repaired_dangling/ { print $2 }')
1365 [ $repaired -eq 32 ] ||
1366 error "(4) Fail to repair dangling reference: $repaired"
1368 echo "'ls' should fail because it will not repair dangling by default"
1369 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(5) ls should fail."
1371 echo "Trigger layout LFSCK to repair dangling reference"
1372 $START_LAYOUT -r -c || error "(6) Fail to start LFSCK for layout!"
1374 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1375 mdd.${MDT_DEV}.lfsck_layout |
1376 awk '/^status/ { print \\\$2 }'" "completed" 6 || return 3
1378 local repaired=$($SHOW_LAYOUT |
1379 awk '/^repaired_dangling/ { print $2 }')
1380 [ $repaired -eq 32 ] ||
1381 error "(7) Fail to repair dangling reference: $repaired"
1383 echo "'ls' should success after layout LFSCK repairing"
1384 ls -ail $DIR/$tdir > /dev/null || error "(8) ls should success."
1386 run_test 14 "LFSCK can repair MDT-object with dangling reference"
1390 echo "If the OST-object referenced by the MDT-object back points"
1391 echo "to some non-exist MDT-object, then the LFSCK should repair"
1392 echo "the OST-object to back point to the right MDT-object."
1398 formatall > /dev/null
1400 setupall > /dev/null
1403 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1405 echo "Inject failure stub to make the OST-object to back point to"
1406 echo "non-exist MDT-object."
1407 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
1409 do_facet ost1 $LCTL set_param fail_loc=0x1611
1410 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1411 cancel_lru_locks osc
1414 do_facet ost1 $LCTL set_param fail_loc=0
1416 echo "stopall to cleanup object cache"
1419 setupall > /dev/null
1421 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1422 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1424 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1425 mdd.${MDT_DEV}.lfsck_layout |
1426 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1428 local repaired=$($SHOW_LAYOUT |
1429 awk '/^repaired_unmatched_pair/ { print $2 }')
1430 [ $repaired -eq 1 ] ||
1431 error "(3) Fail to repair unmatched pair: $repaired"
1433 run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
1437 echo "If the OST-object referenced by the MDT-object back points"
1438 echo "to other MDT-object that doesn't recognize the OST-object,"
1439 echo "then the LFSCK should repair it to back point to the right"
1440 echo "MDT-object (the first one)."
1446 formatall > /dev/null
1448 setupall > /dev/null
1451 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1452 dd if=/dev/zero of=$DIR/$tdir/guard bs=1K count=1
1453 cancel_lru_locks osc
1455 echo "Inject failure stub to make the OST-object to back point to"
1456 echo "other MDT-object"
1458 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
1459 do_facet ost1 $LCTL set_param fail_loc=0x1612
1460 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1461 cancel_lru_locks osc
1464 do_facet ost1 $LCTL set_param fail_loc=0
1466 echo "stopall to cleanup object cache"
1469 setupall > /dev/null
1471 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1472 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1474 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1475 mdd.${MDT_DEV}.lfsck_layout |
1476 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1478 local repaired=$($SHOW_LAYOUT |
1479 awk '/^repaired_unmatched_pair/ { print $2 }')
1480 [ $repaired -eq 1 ] ||
1481 error "(3) Fail to repair unmatched pair: $repaired"
1483 run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
1487 echo "If the OST-object's owner information does not match the owner"
1488 echo "information stored in the MDT-object, then the LFSCK trust the"
1489 echo "MDT-object and update the OST-object's owner information."
1495 formatall > /dev/null
1497 setupall > /dev/null
1500 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1501 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1502 cancel_lru_locks osc
1506 echo "Inject failure stub to skip OST-object owner changing"
1507 #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
1508 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
1509 chown 1.1 $DIR/$tdir/f0
1510 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1512 echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
1515 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1517 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1518 mdd.${MDT_DEV}.lfsck_layout |
1519 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1521 local repaired=$($SHOW_LAYOUT |
1522 awk '/^repaired_inconsistent_owner/ { print $2 }')
1523 [ $repaired -eq 1 ] ||
1524 error "(3) Fail to repair inconsistent owner: $repaired"
1526 run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
1530 echo "If more than one MDT-objects reference the same OST-object,"
1531 echo "and the OST-object only recognizes one MDT-object, then the"
1532 echo "LFSCK should create new OST-objects for such non-recognized"
1539 formatall > /dev/null
1541 setupall > /dev/null
1544 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1546 echo "Inject failure stub to make two MDT-objects to refernce"
1547 echo "the OST-object"
1549 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1550 #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
1551 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1614
1553 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1554 cancel_lru_locks osc
1558 createmany -o $DIR/$tdir/f 1 > /dev/null 2>&1
1560 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1561 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1563 echo "stopall to cleanup object cache"
1566 setupall > /dev/null
1568 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
1569 local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
1570 [ $size -eq 1048576 ] ||
1571 error "(1) f0 (wrong) size should be 1048576, but got $size"
1573 echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
1576 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1578 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1579 mdd.${MDT_DEV}.lfsck_layout |
1580 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 3
1582 local repaired=$($SHOW_LAYOUT |
1583 awk '/^repaired_multiple_referenced/ { print $2 }')
1584 [ $repaired -eq 1 ] ||
1585 error "(4) Fail to repair multiple references: $repaired"
1587 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
1588 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
1589 error "(5) Fail to write f0."
1590 size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
1591 [ $size -eq 1048576 ] ||
1592 error "(6) guard size should be 1048576, but got $size"
1594 run_test 17 "LFSCK can repair multiple references"
1597 [ $MDSCOUNT -lt 2 ] &&
1598 skip "We need at least 2 MDSes for test_18a" && exit 0
1600 [ $OSTCOUNT -lt 2 ] &&
1601 skip "We need at least 2 OSTs for test_18a" && exit 0
1604 echo "The target MDT-object is there, but related stripe information"
1605 echo "is lost or partly lost. The LFSCK should regenerate the missed"
1606 echo "layout EA entries."
1612 formatall > /dev/null
1614 setupall > /dev/null
1617 $LFS mkdir -i 0 $DIR/$tdir/a1
1618 $LFS mkdir -i 1 $DIR/$tdir/a2
1619 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1620 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1621 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1622 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1624 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1626 $LFS path2fid $DIR/$tdir/a1/f1
1627 $LFS getstripe $DIR/$tdir/a1/f1
1628 $LFS path2fid $DIR/$tdir/a2/f2
1629 $LFS getstripe $DIR/$tdir/a2/f2
1631 cancel_lru_locks osc
1633 echo "Inject failure, to make the MDT-object lost its layout EA"
1634 #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
1635 do_facet mds1 $LCTL set_param fail_loc=0x1615
1636 chown 1.1 $DIR/$tdir/a1/f1
1637 do_facet mds2 $LCTL set_param fail_loc=0x1615
1638 chown 1.1 $DIR/$tdir/a2/f2
1641 do_facet mds1 $LCTL set_param fail_loc=0
1642 do_facet mds2 $LCTL set_param fail_loc=0
1644 echo "stopall to cleanup object cache"
1647 setupall > /dev/null
1649 echo "The file size should be incorrect since layout EA is lost"
1650 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1651 [ "$cur_size" != "$saved_size" ] ||
1652 error "(1) Expect incorrect file1 size"
1654 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1655 [ "$cur_size" != "$saved_size" ] ||
1656 error "(2) Expect incorrect file2 size"
1658 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1659 $START_LAYOUT -o || error "(3) Fail to start LFSCK for layout!"
1661 for k in $(seq $MDSCOUNT); do
1662 # The LFSCK status query internal is 30 seconds. For the case
1663 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1664 # time to guarantee the status sync up.
1665 wait_update_facet mds${k} "$LCTL get_param -n \
1666 mdd.$(facet_svc mds${k}).lfsck_layout |
1667 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1668 error "(4) MDS${k} is not the expected 'completed'"
1671 for k in $(seq $OSTCOUNT); do
1672 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1673 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1674 awk '/^status/ { print $2 }')
1675 [ "$cur_status" == "completed" ] ||
1676 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1680 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1681 mdd.$(facet_svc mds${k}).lfsck_layout |
1682 awk '/^repaired_orphan/ { print $2 }')
1683 [ $repaired -eq ${k} ] ||
1684 error "(6) Expect ${k} fixed on mds${k}, but got: $repaired"
1687 $LFS path2fid $DIR/$tdir/a1/f1
1688 $LFS getstripe $DIR/$tdir/a1/f1
1689 $LFS path2fid $DIR/$tdir/a2/f2
1690 $LFS getstripe $DIR/$tdir/a2/f2
1692 echo "The file size should be correct after layout LFSCK scanning"
1693 cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1694 [ "$cur_size" == "$saved_size" ] ||
1695 error "(7) Expect file1 size $saved_size, but got $cur_size"
1697 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1698 [ "$cur_size" == "$saved_size" ] ||
1699 error "(8) Expect file2 size $saved_size, but got $cur_size"
1701 run_test 18a "Find out orphan OST-object and repair it (1)"
1704 [ $MDSCOUNT -lt 2 ] &&
1705 skip "We need at least 2 MDSes for test_18b" && exit 0
1707 [ $OSTCOUNT -lt 2 ] &&
1708 skip "We need at least 2 OSTs for test_18b" && exit 0
1711 echo "The target MDT-object is lost. The LFSCK should re-create the"
1712 echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
1713 echo "can move it back to normal namespace manually."
1719 formatall > /dev/null
1721 setupall > /dev/null
1724 $LFS mkdir -i 0 $DIR/$tdir/a1
1725 $LFS mkdir -i 1 $DIR/$tdir/a2
1726 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1727 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1728 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1729 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1730 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1731 local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
1733 $LFS getstripe $DIR/$tdir/a1/f1
1734 local fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
1736 $LFS getstripe $DIR/$tdir/a2/f2
1738 cancel_lru_locks osc
1740 echo "Inject failure, to simulate the case of missing the MDT-object"
1741 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1742 do_facet mds1 $LCTL set_param fail_loc=0x1616
1743 rm -f $DIR/$tdir/a1/f1
1744 do_facet mds2 $LCTL set_param fail_loc=0x1616
1745 rm -f $DIR/$tdir/a2/f2
1748 do_facet mds1 $LCTL set_param fail_loc=0
1749 do_facet mds2 $LCTL set_param fail_loc=0
1751 echo "stopall to cleanup object cache"
1754 setupall > /dev/null
1756 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1757 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1759 for k in $(seq $MDSCOUNT); do
1760 # The LFSCK status query internal is 30 seconds. For the case
1761 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1762 # time to guarantee the status sync up.
1763 wait_update_facet mds${k} "$LCTL get_param -n \
1764 mdd.$(facet_svc mds${k}).lfsck_layout |
1765 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1766 error "(2) MDS${k} is not the expected 'completed'"
1769 for k in $(seq $OSTCOUNT); do
1770 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1771 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1772 awk '/^status/ { print $2 }')
1773 [ "$cur_status" == "completed" ] ||
1774 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1778 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1779 mdd.$(facet_svc mds${k}).lfsck_layout |
1780 awk '/^repaired_orphan/ { print $2 }')
1781 [ $repaired -eq ${k} ] ||
1782 error "(4) Expect ${k} fixed on mds${k}, but got: $repaired"
1785 echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
1786 mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
1787 error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
1789 mv $MOUNT/.lustre/lost+found/MDT0001/R-${fid2} $DIR/$tdir/a2/f2 ||
1790 error "(6) Fail to move $MOUNT/.lustre/lost+found/MDT0001/R-${fid2}"
1792 $LFS path2fid $DIR/$tdir/a1/f1
1793 $LFS getstripe $DIR/$tdir/a1/f1
1794 $LFS path2fid $DIR/$tdir/a2/f2
1795 $LFS getstripe $DIR/$tdir/a2/f2
1797 echo "The file size should be correct after layout LFSCK scanning"
1798 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1799 [ "$cur_size" == "$saved_size" ] ||
1800 error "(7) Expect file1 size $saved_size, but got $cur_size"
1802 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1803 [ "$cur_size" == "$saved_size" ] ||
1804 error "(8) Expect file2 size $saved_size, but got $cur_size"
1806 run_test 18b "Find out orphan OST-object and repair it (2)"
1809 [ $MDSCOUNT -lt 2 ] &&
1810 skip "We need at least 2 MDSes for test_18c" && exit 0
1812 [ $OSTCOUNT -lt 2 ] &&
1813 skip "We need at least 2 OSTs for test_18c" && exit 0
1816 echo "The target MDT-object is lost, and the OST-object FID is missing."
1817 echo "The LFSCK should re-create the MDT-object with new FID under the "
1818 echo "directory .lustre/lost+found/MDTxxxx."
1824 formatall > /dev/null
1826 setupall > /dev/null
1829 $LFS mkdir -i 0 $DIR/$tdir/a1
1830 $LFS mkdir -i 1 $DIR/$tdir/a2
1831 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1832 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1834 echo "Inject failure, to simulate the case of missing parent FID"
1835 #define OBD_FAIL_LFSCK_NOPFID 0x1617
1836 do_facet ost1 $LCTL set_param fail_loc=0x1617
1837 do_facet ost2 $LCTL set_param fail_loc=0x1617
1839 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1840 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1841 $LFS getstripe $DIR/$tdir/a1/f1
1842 $LFS getstripe $DIR/$tdir/a2/f2
1844 cancel_lru_locks osc
1846 echo "Inject failure, to simulate the case of missing the MDT-object"
1847 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1848 do_facet mds1 $LCTL set_param fail_loc=0x1616
1849 rm -f $DIR/$tdir/a1/f1
1850 do_facet mds2 $LCTL set_param fail_loc=0x1616
1851 rm -f $DIR/$tdir/a2/f2
1854 do_facet mds1 $LCTL set_param fail_loc=0
1855 do_facet mds2 $LCTL set_param fail_loc=0
1857 echo "stopall to cleanup object cache"
1860 setupall > /dev/null
1862 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1863 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1865 for k in $(seq $MDSCOUNT); do
1866 # The LFSCK status query internal is 30 seconds. For the case
1867 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1868 # time to guarantee the status sync up.
1869 wait_update_facet mds${k} "$LCTL get_param -n \
1870 mdd.$(facet_svc mds${k}).lfsck_layout |
1871 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1872 error "(2) MDS${k} is not the expected 'completed'"
1875 for k in $(seq $OSTCOUNT); do
1876 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1877 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1878 awk '/^status/ { print $2 }')
1879 [ "$cur_status" == "completed" ] ||
1880 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1883 local repaired=$(do_facet mds1 $LCTL get_param -n \
1884 mdd.$(facet_svc mds1).lfsck_layout |
1885 awk '/^repaired_orphan/ { print $2 }')
1886 [ $repaired -eq 3 ] ||
1887 error "(4) Expect 3 fixed on mds1, but got: $repaired"
1889 repaired=$(do_facet mds2 $LCTL get_param -n \
1890 mdd.$(facet_svc mds2).lfsck_layout |
1891 awk '/^repaired_orphan/ { print $2 }')
1892 [ $repaired -eq 0 ] ||
1893 error "(5) Expect 0 fixed on mds2, but got: $repaired"
1895 echo "There should be some stub under .lustre/lost+found/MDT0001/"
1896 ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
1897 error "(6) .lustre/lost+found/MDT0001/ should be empty"
1899 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1900 ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
1901 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1903 run_test 18c "Find out orphan OST-object and repair it (3)"
1907 echo "The target MDT-object layout EA slot is occpuied by some new"
1908 echo "created OST-object when repair dangling reference case. Such"
1909 echo "conflict OST-object has never been modified. Then when found"
1910 echo "the orphan OST-object, LFSCK will replace it with the orphan"
1917 formatall > /dev/null
1919 setupall > /dev/null
1921 mkdir -p $DIR/$tdir/a1
1922 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1923 echo "guard" > $DIR/$tdir/a1/f1
1924 echo "foo" > $DIR/$tdir/a1/f2
1925 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1926 $LFS path2fid $DIR/$tdir/a1/f1
1927 $LFS getstripe $DIR/$tdir/a1/f1
1928 $LFS path2fid $DIR/$tdir/a1/f2
1929 $LFS getstripe $DIR/$tdir/a1/f2
1931 cancel_lru_locks osc
1933 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1934 echo "to reference the same OST-object (which is f1's OST-obejct)."
1935 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1936 echo "dangling reference case, but f2's old OST-object is there."
1939 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1940 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1941 chown 1.1 $DIR/$tdir/a1/f2
1942 rm -f $DIR/$tdir/a1/f1
1945 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1947 echo "stopall to cleanup object cache"
1950 setupall > /dev/null
1952 echo "The file size should be incorrect since dangling referenced"
1953 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1954 [ "$cur_size" != "$saved_size" ] ||
1955 error "(1) Expect incorrect file2 size"
1957 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1958 $START_LAYOUT -o -c || error "(2) Fail to start LFSCK for layout!"
1960 for k in $(seq $MDSCOUNT); do
1961 # The LFSCK status query internal is 30 seconds. For the case
1962 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1963 # time to guarantee the status sync up.
1964 wait_update_facet mds${k} "$LCTL get_param -n \
1965 mdd.$(facet_svc mds${k}).lfsck_layout |
1966 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1967 error "(3) MDS${k} is not the expected 'completed'"
1970 for k in $(seq $OSTCOUNT); do
1971 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1972 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1973 awk '/^status/ { print $2 }')
1974 [ "$cur_status" == "completed" ] ||
1975 error "(4) OST${k} Expect 'completed', but got '$cur_status'"
1978 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1979 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1980 awk '/^repaired_orphan/ { print $2 }')
1981 [ $repaired -eq 1 ] ||
1982 error "(5) Expect 1 orphan has been fixed, but got: $repaired"
1984 echo "The file size should be correct after layout LFSCK scanning"
1985 cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1986 [ "$cur_size" == "$saved_size" ] ||
1987 error "(6) Expect file2 size $saved_size, but got $cur_size"
1989 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1990 ls -ail $MOUNT/.lustre/lost+found/MDT0000/ &&
1991 error "(7) .lustre/lost+found/MDT0000/ should be empty"
1993 echo "The LFSCK should find back the original data."
1994 cat $DIR/$tdir/a1/f2
1995 $LFS path2fid $DIR/$tdir/a1/f2
1996 $LFS getstripe $DIR/$tdir/a1/f2
1998 run_test 18d "Find out orphan OST-object and repair it (4)"
2002 echo "The target MDT-object layout EA slot is occpuied by some new"
2003 echo "created OST-object when repair dangling reference case. Such"
2004 echo "conflict OST-object has been modified by others. To keep the"
2005 echo "new data, the LFSCK will create a new file to refernece this"
2006 echo "old orphan OST-object."
2012 formatall > /dev/null
2014 setupall > /dev/null
2016 mkdir -p $DIR/$tdir/a1
2017 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
2018 echo "guard" > $DIR/$tdir/a1/f1
2019 echo "foo" > $DIR/$tdir/a1/f2
2020 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2021 $LFS path2fid $DIR/$tdir/a1/f1
2022 $LFS getstripe $DIR/$tdir/a1/f1
2023 $LFS path2fid $DIR/$tdir/a1/f2
2024 $LFS getstripe $DIR/$tdir/a1/f2
2026 cancel_lru_locks osc
2028 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
2029 echo "to reference the same OST-object (which is f1's OST-obejct)."
2030 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
2031 echo "dangling reference case, but f2's old OST-object is there."
2034 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
2035 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
2036 chown 1.1 $DIR/$tdir/a1/f2
2037 rm -f $DIR/$tdir/a1/f1
2040 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2042 echo "stopall to cleanup object cache"
2045 setupall > /dev/null
2047 echo "The file size should be incorrect since dangling referenced"
2048 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2049 [ "$cur_size" != "$saved_size" ] ||
2050 error "(1) Expect incorrect file2 size"
2052 #define OBD_FAIL_LFSCK_DELAY3 0x1602
2053 do_facet $SINGLEMDS $LCTL set_param fail_val=10
2054 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
2056 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
2057 $START_LAYOUT -o -c || error "(2) Fail to start LFSCK for layout!"
2059 wait_update_facet mds1 "$LCTL get_param -n \
2060 mdd.$(facet_svc mds1).lfsck_layout |
2061 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
2062 error "(3) MDS1 is not the expected 'scanning-phase2'"
2064 # to guarantee all updates are synced.
2068 echo "Write new data to f2 to modify the new created OST-object."
2069 echo "dummy" >> $DIR/$tdir/a1/f2
2071 do_facet $SINGLEMDS $LCTL set_param fail_val=0
2072 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2074 for k in $(seq $MDSCOUNT); do
2075 # The LFSCK status query internal is 30 seconds. For the case
2076 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
2077 # time to guarantee the status sync up.
2078 wait_update_facet mds${k} "$LCTL get_param -n \
2079 mdd.$(facet_svc mds${k}).lfsck_layout |
2080 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
2081 error "(4) MDS${k} is not the expected 'completed'"
2084 for k in $(seq $OSTCOUNT); do
2085 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
2086 obdfilter.$(facet_svc ost${k}).lfsck_layout |
2087 awk '/^status/ { print $2 }')
2088 [ "$cur_status" == "completed" ] ||
2089 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
2092 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
2093 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
2094 awk '/^repaired_orphan/ { print $2 }')
2095 [ $repaired -eq 1 ] ||
2096 error "(6) Expect 1 orphan has been fixed, but got: $repaired"
2098 echo "There should be stub file under .lustre/lost+found/MDT0000/"
2099 local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
2101 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
2103 echo "The stub file should keep the original f2 data"
2104 cur_size=$(ls -il $cname | awk '{ print $6 }')
2105 [ "$cur_size" == "$saved_size" ] ||
2106 error "(8) Expect file2 size $saved_size, but got $cur_size"
2109 $LFS path2fid $cname
2110 $LFS getstripe $cname
2112 echo "The f2 should contains new data."
2113 cat $DIR/$tdir/a1/f2
2114 $LFS path2fid $DIR/$tdir/a1/f2
2115 $LFS getstripe $DIR/$tdir/a1/f2
2117 run_test 18e "Find out orphan OST-object and repair it (5)"
2123 formatall > /dev/null
2125 setupall > /dev/null
2128 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2130 echo "foo" > $DIR/$tdir/a0
2131 echo "guard" > $DIR/$tdir/a1
2133 cancel_lru_locks osc
2134 umount_client $MOUNT || error "(1) Fail to stop client!"
2135 mount_client $MOUNT || error "(2) Fail to start client!"
2137 echo "Inject failure, then client will offer wrong parent FID when read"
2138 do_facet ost1 $LCTL set_param -n \
2139 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2140 #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
2141 $LCTL set_param fail_loc=0x1619
2143 echo "Read RPC with wrong parent FID should be denied"
2144 cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
2145 $LCTL set_param fail_loc=0
2147 run_test 19a "OST-object inconsistency self detect"
2153 formatall > /dev/null
2155 setupall > /dev/null
2158 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2160 echo "Inject failure stub to make the OST-object to back point to"
2161 echo "non-exist MDT-object"
2163 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
2164 do_facet ost1 $LCTL set_param fail_loc=0x1611
2165 echo "foo" > $DIR/$tdir/f0
2166 cancel_lru_locks osc
2169 do_facet ost1 $LCTL set_param fail_loc=0
2171 echo "Nothing should be fixed since self detect and repair is disabled"
2172 local repaired=$(do_facet ost1 $LCTL get_param -n \
2173 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2174 awk '/^repaired/ { print $2 }')
2175 [ $repaired -eq 0 ] ||
2176 error "(1) Expected 0 repaired, but got $repaired"
2178 echo "Read RPC with right parent FID should be accepted,"
2179 echo "and cause parent FID on OST to be fixed"
2181 do_facet ost1 $LCTL set_param -n \
2182 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2183 cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
2185 repaired=$(do_facet ost1 $LCTL get_param -n \
2186 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2187 awk '/^repaired/ { print $2 }')
2188 [ $repaired -eq 1 ] ||
2189 error "(3) Expected 1 repaired, but got $repaired"
2191 run_test 19b "OST-object inconsistency self repair"
2193 $LCTL set_param debug=-lfsck > /dev/null || true
2195 # restore MDS/OST size
2196 MDSSIZE=${SAVED_MDSSIZE}
2197 OSTSIZE=${SAVED_OSTSIZE}
2199 # cleanup the system at last