3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT"
11 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
12 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
14 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
15 . $LUSTRE/tests/test-framework.sh
17 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
20 require_dsh_mds || exit 0
22 MCREATE=${MCREATE:-mcreate}
23 SAVED_MDSSIZE=${MDSSIZE}
24 SAVED_OSTSIZE=${OSTSIZE}
25 # use small MDS + OST size to speed formatting time
26 # do not use too small MDSSIZE/OSTSIZE, which affect the default journal size
30 check_and_setup_lustre
32 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.60) ]] &&
33 skip "Need MDS version at least 2.3.60" && check_and_cleanup_lustre &&
36 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
37 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
39 [[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] &&
40 ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19"
44 $LCTL set_param debug=+lfsck > /dev/null || true
46 MDT_DEV="${FSNAME}-MDT0000"
47 OST_DEV="${FSNAME}-OST0000"
48 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
49 START_NAMESPACE="do_facet $SINGLEMDS \
50 $LCTL lfsck_start -M ${MDT_DEV} -t namespace"
51 START_LAYOUT="do_facet $SINGLEMDS \
52 $LCTL lfsck_start -M ${MDT_DEV} -t layout"
53 START_LAYOUT_ON_OST="do_facet ost1 $LCTL lfsck_start -M ${OST_DEV} -t layout"
54 STOP_LFSCK="do_facet $SINGLEMDS $LCTL lfsck_stop -M ${MDT_DEV}"
55 SHOW_NAMESPACE="do_facet $SINGLEMDS \
56 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace"
57 SHOW_LAYOUT="do_facet $SINGLEMDS \
58 $LCTL get_param -n mdd.${MDT_DEV}.lfsck_layout"
59 SHOW_LAYOUT_ON_OST="do_facet ost1 \
60 $LCTL get_param -n obdfilter.${OST_DEV}.lfsck_layout"
61 MOUNT_OPTS_SCRUB="-o user_xattr"
62 MOUNT_OPTS_NOSCRUB="-o user_xattr,noscrub"
75 if [ ! -z $igif ]; then
76 #define OBD_FAIL_FID_IGIF 0x1504
77 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1504
80 echo "preparing... ${nfiles} * ${ndirs} files will be created."
82 cp $LUSTRE/tests/*.sh $DIR/
83 for ((i = 0; i < ${ndirs}; i++)); do
84 mkdir $DIR/$tdir/d${i}
85 touch $DIR/$tdir/f${i}
86 for ((j = 0; j < ${nfiles}; j++)); do
87 touch $DIR/$tdir/d${i}/f${j}
89 mkdir $DIR/$tdir/e${i}
92 if [ ! -z $igif ]; then
93 touch $DIR/$tdir/dummy
94 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
98 cleanup_mount $MOUNT > /dev/null || error "Fail to stop client!"
99 echo "stop $SINGLEMDS"
100 stop $SINGLEMDS > /dev/null || error "Fail to stop MDS!"
105 echo "start $SINGLEMDS"
106 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
107 error "(1) Fail to start MDS!"
109 #define OBD_FAIL_LFSCK_DELAY1 0x1600
110 do_facet $SINGLEMDS $LCTL set_param fail_val=3
111 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
112 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
114 $SHOW_NAMESPACE || error "Fail to monitor LFSCK (3)"
116 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
117 [ "$STATUS" == "scanning-phase1" ] ||
118 error "(4) Expect 'scanning-phase1', but got '$STATUS'"
120 $STOP_LFSCK || error "(5) Fail to stop LFSCK!"
122 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
123 [ "$STATUS" == "stopped" ] ||
124 error "(6) Expect 'stopped', but got '$STATUS'"
126 $START_NAMESPACE || error "(7) Fail to start LFSCK for namespace!"
128 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
129 [ "$STATUS" == "scanning-phase1" ] ||
130 error "(8) Expect 'scanning-phase1', but got '$STATUS'"
132 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
133 do_facet $SINGLEMDS $LCTL set_param fail_val=0
134 wait_update_facet $SINGLEMDS \
135 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
136 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
137 error "(9) unexpected status"
139 local repaired=$($SHOW_NAMESPACE |
140 awk '/^updated_phase1/ { print $2 }')
141 [ $repaired -eq 0 ] ||
142 error "(10) Expect nothing to be repaired, but got: $repaired"
144 local scanned1=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
145 $START_NAMESPACE -r || error "(11) Fail to reset LFSCK!"
146 wait_update_facet $SINGLEMDS \
147 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace | \
148 awk '/^status/ { print \\\$2 }'" "completed" 20 || \
149 error "(12) unexpected status"
151 local scanned2=$($SHOW_NAMESPACE | awk '/^success_count/ { print $2 }')
152 [ $((scanned1 + 1)) -eq $scanned2 ] ||
153 error "(13) Expect success $((scanned1 + 1)), but got $scanned2"
155 echo "stopall, should NOT crash LU-3649"
158 run_test 0 "Control LFSCK manually"
161 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
162 skip "OI Scrub not implemented for ZFS" && return
165 echo "start $SINGLEMDS"
166 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
167 error "(1) Fail to start MDS!"
169 mount_client $MOUNT || error "(2) Fail to start client!"
171 #define OBD_FAIL_FID_INDIR 0x1501
172 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1501
173 touch $DIR/$tdir/dummy
175 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
177 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
180 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
181 [ "$STATUS" == "completed" ] ||
182 error "(4) Expect 'completed', but got '$STATUS'"
184 local repaired=$($SHOW_NAMESPACE |
185 awk '/^updated_phase1/ { print $2 }')
186 [ $repaired -eq 1 ] ||
187 error "(5) Fail to repair crashed FID-in-dirent: $repaired"
189 mount_client $MOUNT || error "(6) Fail to start client!"
191 #define OBD_FAIL_FID_LOOKUP 0x1505
192 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
193 ls $DIR/$tdir/ > /dev/null || error "(7) no FID-in-dirent."
195 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
197 run_test 1a "LFSCK can find out and repair crashed FID-in-dirent"
201 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
202 skip "OI Scrub not implemented for ZFS" && return
205 echo "start $SINGLEMDS"
206 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
207 error "(1) Fail to start MDS!"
209 mount_client $MOUNT || error "(2) Fail to start client!"
211 #define OBD_FAIL_FID_INLMA 0x1502
212 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1502
213 touch $DIR/$tdir/dummy
215 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
217 #define OBD_FAIL_FID_NOLMA 0x1506
218 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1506
219 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
222 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
223 [ "$STATUS" == "completed" ] ||
224 error "(4) Expect 'completed', but got '$STATUS'"
226 local repaired=$($SHOW_NAMESPACE |
227 awk '/^updated_phase1/ { print $2 }')
228 [ $repaired -eq 1 ] ||
229 error "(5) Fail to repair missed FID-in-LMA: $repaired"
231 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
232 mount_client $MOUNT || error "(6) Fail to start client!"
234 #define OBD_FAIL_FID_LOOKUP 0x1505
235 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
236 stat $DIR/$tdir/dummy > /dev/null || error "(7) no FID-in-LMA."
238 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
240 run_test 1b "LFSCK can find out and repair missed FID-in-LMA"
244 echo "start $SINGLEMDS"
245 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
246 error "(1) Fail to start MDS!"
248 mount_client $MOUNT || error "(2) Fail to start client!"
250 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
251 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
252 touch $DIR/$tdir/dummy
254 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
256 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
259 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
260 [ "$STATUS" == "completed" ] ||
261 error "(4) Expect 'completed', but got '$STATUS'"
263 local repaired=$($SHOW_NAMESPACE |
264 awk '/^updated_phase1/ { print $2 }')
265 [ $repaired -eq 1 ] ||
266 error "(5) Fail to repair crashed linkEA: $repaired"
268 mount_client $MOUNT || error "(6) Fail to start client!"
270 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
271 error "(7) Fail to stat $DIR/$tdir/dummy"
273 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
274 local dummyname=$($LFS fid2path $DIR $dummyfid)
275 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
276 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
278 run_test 2a "LFSCK can find out and repair crashed linkEA entry"
283 echo "start $SINGLEMDS"
284 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
285 error "(1) Fail to start MDS!"
287 mount_client $MOUNT || error "(2) Fail to start client!"
289 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
290 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
291 touch $DIR/$tdir/dummy
293 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
295 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
298 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
299 [ "$STATUS" == "completed" ] ||
300 error "(4) Expect 'completed', but got '$STATUS'"
302 local repaired=$($SHOW_NAMESPACE |
303 awk '/^updated_phase2/ { print $2 }')
304 [ $repaired -eq 1 ] ||
305 error "(5) Fail to repair crashed linkEA: $repaired"
307 mount_client $MOUNT || error "(6) Fail to start client!"
309 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
310 error "(7) Fail to stat $DIR/$tdir/dummy"
312 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
313 local dummyname=$($LFS fid2path $DIR $dummyfid)
314 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
315 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
317 run_test 2b "LFSCK can find out and remove invalid linkEA entry"
322 echo "start $SINGLEMDS"
323 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
324 error "(1) Fail to start MDS!"
326 mount_client $MOUNT || error "(2) Fail to start client!"
328 #define OBD_FAIL_LFSCK_LINKEA_MORE2 0x1605
329 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1605
330 touch $DIR/$tdir/dummy
332 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
334 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
337 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
338 [ "$STATUS" == "completed" ] ||
339 error "(4) Expect 'completed', but got '$STATUS'"
341 local repaired=$($SHOW_NAMESPACE |
342 awk '/^updated_phase2/ { print $2 }')
343 [ $repaired -eq 1 ] ||
344 error "(5) Fail to repair crashed linkEA: $repaired"
346 mount_client $MOUNT || error "(6) Fail to start client!"
348 stat $DIR/$tdir/dummy | grep "Links: 1" > /dev/null ||
349 error "(7) Fail to stat $DIR/$tdir/dummy"
351 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
352 local dummyname=$($LFS fid2path $DIR $dummyfid)
353 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
354 error "(8) Fail to repair linkEA: $dummyfid $dummyname"
356 run_test 2c "LFSCK can find out and remove repeated linkEA entry"
360 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
361 skip "OI Scrub not implemented for ZFS" && return
364 mds_backup_restore $SINGLEMDS || error "(1) Fail to backup/restore!"
365 echo "start $SINGLEMDS with disabling OI scrub"
366 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
367 error "(2) Fail to start MDS!"
369 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
370 [ "$STATUS" == "init" ] ||
371 error "(3) Expect 'init', but got '$STATUS'"
373 #define OBD_FAIL_LFSCK_DELAY2 0x1601
374 do_facet $SINGLEMDS $LCTL set_param fail_val=1
375 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
376 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
379 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
380 [ "$STATUS" == "scanning-phase1" ] ||
381 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
383 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
384 [ "$FLAGS" == "inconsistent" ] ||
385 error "(6) Expect 'inconsistent', but got '$FLAGS'"
387 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
388 do_facet $SINGLEMDS $LCTL set_param fail_val=0
390 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
391 [ "$STATUS" == "completed" ] ||
392 error "(7) Expect 'completed', but got '$STATUS'"
394 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
395 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
397 local repaired=$($SHOW_NAMESPACE |
398 awk '/^updated_phase1/ { print $2 }')
399 [ $repaired -ge 9 ] ||
400 error "(9) Fail to repair crashed linkEA: $repaired"
402 mount_client $MOUNT || error "(10) Fail to start client!"
404 #define OBD_FAIL_FID_LOOKUP 0x1505
405 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
406 ls $DIR/$tdir/ > /dev/null || error "(11) no FID-in-dirent."
408 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
410 run_test 4 "FID-in-dirent can be rebuilt after MDT file-level backup/restore"
414 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
415 skip "OI Scrub not implemented for ZFS" && return
418 mds_backup_restore $SINGLEMDS 1 || error "(1) Fail to backup/restore!"
419 echo "start $SINGLEMDS with disabling OI scrub"
420 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_NOSCRUB > /dev/null ||
421 error "(2) Fail to start MDS!"
423 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
424 [ "$STATUS" == "init" ] ||
425 error "(3) Expect 'init', but got '$STATUS'"
427 #define OBD_FAIL_LFSCK_DELAY2 0x1601
428 do_facet $SINGLEMDS $LCTL set_param fail_val=1
429 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
430 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
433 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
434 [ "$STATUS" == "scanning-phase1" ] ||
435 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
437 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
438 [ "$FLAGS" == "inconsistent,upgrade" ] ||
439 error "(6) Expect 'inconsistent,upgrade', but got '$FLAGS'"
441 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
442 do_facet $SINGLEMDS $LCTL set_param fail_val=0
444 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
445 [ "$STATUS" == "completed" ] ||
446 error "(7) Expect 'completed', but got '$STATUS'"
448 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
449 [ -z "$FLAGS" ] || error "(8) Expect empty flags, but got '$FLAGS'"
451 local repaired=$($SHOW_NAMESPACE |
452 awk '/^updated_phase1/ { print $2 }')
453 [ $repaired -ge 2 ] ||
454 error "(9) Fail to repair crashed linkEA: $repaired"
456 mount_client $MOUNT || error "(10) Fail to start client!"
458 #define OBD_FAIL_FID_LOOKUP 0x1505
459 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1505
460 stat $DIR/$tdir/dummy > /dev/null || error "(11) no FID-in-LMA."
462 ls $DIR/$tdir/ > /dev/null || error "(12) no FID-in-dirent."
464 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
465 local dummyfid=$($LFS path2fid $DIR/$tdir/dummy)
466 local dummyname=$($LFS fid2path $DIR $dummyfid)
467 [ "$dummyname" == "$DIR/$tdir/dummy" ] ||
468 error "(13) Fail to generate linkEA: $dummyfid $dummyname"
470 run_test 5 "LFSCK can handle IFIG object upgrading"
474 echo "start $SINGLEMDS"
475 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
476 error "(1) Fail to start MDS!"
478 #define OBD_FAIL_LFSCK_DELAY1 0x1600
479 do_facet $SINGLEMDS $LCTL set_param fail_val=1
480 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
481 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
483 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
484 [ "$STATUS" == "scanning-phase1" ] ||
485 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
487 # Sleep 3 sec to guarantee at least one object processed by LFSCK
489 # Fail the LFSCK to guarantee there is at least one checkpoint
490 #define OBD_FAIL_LFSCK_FATAL1 0x1608
491 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001608
493 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
494 [ "$STATUS" == "failed" ] ||
495 error "(4) Expect 'failed', but got '$STATUS'"
497 local POSITION0=$($SHOW_NAMESPACE |
498 awk '/^last_checkpoint_position/ { print $2 }' |
501 #define OBD_FAIL_LFSCK_DELAY1 0x1600
502 do_facet $SINGLEMDS $LCTL set_param fail_val=1
503 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
504 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
506 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
507 [ "$STATUS" == "scanning-phase1" ] ||
508 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
510 local POSITION1=$($SHOW_NAMESPACE |
511 awk '/^latest_start_position/ { print $2 }' |
513 [ $POSITION0 -lt $POSITION1 ] ||
514 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
516 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
517 do_facet $SINGLEMDS $LCTL set_param fail_val=0
519 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
520 [ "$STATUS" == "completed" ] ||
521 error "(8) Expect 'completed', but got '$STATUS'"
523 run_test 6a "LFSCK resumes from last checkpoint (1)"
527 echo "start $SINGLEMDS"
528 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
529 error "(1) Fail to start MDS!"
531 #define OBD_FAIL_LFSCK_DELAY2 0x1601
532 do_facet $SINGLEMDS $LCTL set_param fail_val=1
533 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
534 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
536 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
537 [ "$STATUS" == "scanning-phase1" ] ||
538 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
540 # Sleep 3 sec to guarantee at least one object processed by LFSCK
542 # Fail the LFSCK to guarantee there is at least one checkpoint
543 #define OBD_FAIL_LFSCK_FATAL2 0x1609
544 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
546 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
547 [ "$STATUS" == "failed" ] ||
548 error "(4) Expect 'failed', but got '$STATUS'"
550 local POSITION0=$($SHOW_NAMESPACE |
551 awk '/^last_checkpoint_position/ { print $4 }')
553 #define OBD_FAIL_LFSCK_DELAY2 0x1601
554 do_facet $SINGLEMDS $LCTL set_param fail_val=1
555 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
556 $START_NAMESPACE || error "(5) Fail to start LFSCK for namespace!"
558 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
559 [ "$STATUS" == "scanning-phase1" ] ||
560 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
562 local POSITION1=$($SHOW_NAMESPACE |
563 awk '/^latest_start_position/ { print $4 }')
564 if [ $POSITION0 -gt $POSITION1 ]; then
565 [ $POSITION1 -eq 0 -a $POSITION0 -eq $((POSITION1 + 1)) ] ||
566 error "(7) Expect larger than: $POSITION0, but got $POSITION1"
569 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
570 do_facet $SINGLEMDS $LCTL set_param fail_val=0
572 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
573 [ "$STATUS" == "completed" ] ||
574 error "(8) Expect 'completed', but got '$STATUS'"
576 run_test 6b "LFSCK resumes from last checkpoint (2)"
581 echo "start $SINGLEMDS"
582 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
583 error "(1) Fail to start MDS!"
585 #define OBD_FAIL_LFSCK_DELAY2 0x1601
586 do_facet $SINGLEMDS $LCTL set_param fail_val=1
587 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
588 $START_NAMESPACE || error "(2) Fail to start LFSCK for namespace!"
590 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
591 [ "$STATUS" == "scanning-phase1" ] ||
592 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
594 # Sleep 3 sec to guarantee at least one object processed by LFSCK
596 echo "stop $SINGLEMDS"
597 stop $SINGLEMDS > /dev/null || error "(4) Fail to stop MDS!"
599 echo "start $SINGLEMDS"
600 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
601 error "(5) Fail to start MDS!"
603 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
604 [ "$STATUS" == "scanning-phase1" ] ||
605 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
607 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
608 do_facet $SINGLEMDS $LCTL set_param fail_val=0
610 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
611 [ "$STATUS" == "completed" ] ||
612 error "(7) Expect 'completed', but got '$STATUS'"
614 run_test 7a "non-stopped LFSCK should auto restarts after MDS remount (1)"
619 echo "start $SINGLEMDS"
620 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
621 error "(1) Fail to start MDS!"
623 mount_client $MOUNT || error "(2) Fail to start client!"
625 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
626 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
627 for ((i = 0; i < 20; i++)); do
628 touch $DIR/$tdir/dummy${i}
631 #define OBD_FAIL_LFSCK_DELAY3 0x1602
632 do_facet $SINGLEMDS $LCTL set_param fail_val=1
633 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
634 $START_NAMESPACE || error "(3) Fail to start LFSCK for namespace!"
637 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
638 [ "$STATUS" == "scanning-phase2" ] ||
639 error "(4) Expect 'scanning-phase2', but got '$STATUS'"
641 echo "stop $SINGLEMDS"
642 stop $SINGLEMDS > /dev/null || error "(5) Fail to stop MDS!"
644 echo "start $SINGLEMDS"
645 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
646 error "(6) Fail to start MDS!"
648 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
649 [ "$STATUS" == "scanning-phase2" ] ||
650 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
652 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
653 do_facet $SINGLEMDS $LCTL set_param fail_val=0
655 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
656 [ "$STATUS" == "completed" ] ||
657 error "(8) Expect 'completed', but got '$STATUS'"
659 run_test 7b "non-stopped LFSCK should auto restarts after MDS remount (2)"
664 echo "start $SINGLEMDS"
665 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
666 error "(1) Fail to start MDS!"
668 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
669 [ "$STATUS" == "init" ] ||
670 error "(2) Expect 'init', but got '$STATUS'"
672 mount_client $MOUNT || error "(3) Fail to start client!"
674 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
675 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
676 mkdir $DIR/$tdir/crashed
678 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
679 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
680 for ((i = 0; i < 5; i++)); do
681 touch $DIR/$tdir/dummy${i}
684 #define OBD_FAIL_LFSCK_DELAY2 0x1601
685 do_facet $SINGLEMDS $LCTL set_param fail_val=2
686 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
687 $START_NAMESPACE || error "(4) Fail to start LFSCK for namespace!"
689 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
690 [ "$STATUS" == "scanning-phase1" ] ||
691 error "(5) Expect 'scanning-phase1', but got '$STATUS'"
693 $STOP_LFSCK || error "(6) Fail to stop LFSCK!"
695 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
696 [ "$STATUS" == "stopped" ] ||
697 error "(7) Expect 'stopped', but got '$STATUS'"
699 $START_NAMESPACE || error "(8) Fail to start LFSCK for namespace!"
701 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
702 [ "$STATUS" == "scanning-phase1" ] ||
703 error "(9) Expect 'scanning-phase1', but got '$STATUS'"
705 #define OBD_FAIL_LFSCK_FATAL2 0x1609
706 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x80001609
708 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
709 [ "$STATUS" == "failed" ] ||
710 error "(10) Expect 'failed', but got '$STATUS'"
712 #define OBD_FAIL_LFSCK_DELAY1 0x1600
713 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1600
714 $START_NAMESPACE || error "(11) Fail to start LFSCK for namespace!"
716 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
717 [ "$STATUS" == "scanning-phase1" ] ||
718 error "(12) Expect 'scanning-phase1', but got '$STATUS'"
720 #define OBD_FAIL_LFSCK_CRASH 0x160a
721 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160a
724 echo "stop $SINGLEMDS"
725 stop $SINGLEMDS > /dev/null || error "(13) Fail to stop MDS!"
727 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
728 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
730 echo "start $SINGLEMDS"
731 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
732 error "(14) Fail to start MDS!"
734 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
735 [ "$STATUS" == "crashed" ] ||
736 error "(15) Expect 'crashed', but got '$STATUS'"
738 #define OBD_FAIL_LFSCK_DELAY2 0x1601
739 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1601
740 $START_NAMESPACE || error "(16) Fail to start LFSCK for namespace!"
742 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
743 [ "$STATUS" == "scanning-phase1" ] ||
744 error "(17) Expect 'scanning-phase1', but got '$STATUS'"
746 echo "stop $SINGLEMDS"
747 stop $SINGLEMDS > /dev/null || error "(18) Fail to stop MDS!"
749 #define OBD_FAIL_LFSCK_NO_AUTO 0x160b
750 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160b
752 echo "start $SINGLEMDS"
753 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
754 error "(19) Fail to start MDS!"
756 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
757 [ "$STATUS" == "paused" ] ||
758 error "(20) Expect 'paused', but got '$STATUS'"
760 #define OBD_FAIL_LFSCK_DELAY3 0x1602
761 do_facet $SINGLEMDS $LCTL set_param fail_val=2
762 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
764 $START_NAMESPACE || error "(21) Fail to start LFSCK for namespace!"
766 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
767 [ "$STATUS" == "scanning-phase2" ] ||
768 error "(22) Expect 'scanning-phase2', but got '$STATUS'"
770 local FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
771 [ "$FLAGS" == "scanned-once,inconsistent" ] ||
772 error "(23) Expect 'scanned-once,inconsistent',but got '$FLAGS'"
774 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
775 do_facet $SINGLEMDS $LCTL set_param fail_val=0
777 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
778 [ "$STATUS" == "completed" ] ||
779 error "(24) Expect 'completed', but got '$STATUS'"
781 FLAGS=$($SHOW_NAMESPACE | awk '/^flags/ { print $2 }')
782 [ -z "$FLAGS" ] || error "(25) Expect empty flags, but got '$FLAGS'"
785 run_test 8 "LFSCK state machine"
788 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
789 skip "Testing on UP system, the speed may be inaccurate."
794 echo "start $SINGLEMDS"
795 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
796 error "(1) Fail to start MDS!"
798 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
799 [ "$STATUS" == "init" ] ||
800 error "(2) Expect 'init', but got '$STATUS'"
802 local BASE_SPEED1=100
804 $START_NAMESPACE -s $BASE_SPEED1 || error "(3) Fail to start LFSCK!"
807 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
808 [ "$STATUS" == "scanning-phase1" ] ||
809 error "(3) Expect 'scanning-phase1', but got '$STATUS'"
811 local SPEED=$($SHOW_NAMESPACE |
812 awk '/^average_speed_phase1/ { print $2 }')
814 # There may be time error, normally it should be less than 2 seconds.
815 # We allow another 20% schedule error.
817 # MAX_MARGIN = 1.2 = 12 / 10
818 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
819 RUN_TIME1 * 12 / 10))
820 [ $SPEED -lt $MAX_SPEED ] ||
821 error "(4) Got speed $SPEED, expected less than $MAX_SPEED"
824 local BASE_SPEED2=300
826 do_facet $SINGLEMDS \
827 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
830 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase1/ { print $2 }')
831 # MIN_MARGIN = 0.8 = 8 / 10
832 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
833 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
834 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
835 [ $SPEED -gt $MIN_SPEED ] ||
836 error "(5) Got speed $SPEED, expected more than $MIN_SPEED"
838 # MAX_MARGIN = 1.2 = 12 / 10
839 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
840 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
841 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
842 [ $SPEED -lt $MAX_SPEED ] ||
843 error "(6) Got speed $SPEED, expected less than $MAX_SPEED"
845 do_facet $SINGLEMDS \
846 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
848 wait_update_facet $SINGLEMDS \
849 "$LCTL get_param -n mdd.${MDT_DEV}.lfsck_namespace|\
850 awk '/^status/ { print \\\$2 }'" "completed" 30 ||
851 error "(7) Failed to get expected 'completed'"
853 run_test 9a "LFSCK speed control (1)"
856 if [ -z "$(grep "processor.*: 1" /proc/cpuinfo)" ]; then
857 skip "Testing on UP system, the speed may be inaccurate."
862 echo "start $SINGLEMDS"
863 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
864 error "(1) Fail to start MDS!"
866 mount_client $MOUNT || error "(2) Fail to start client!"
868 echo "Another preparing... 50 * 50 files (with error) will be created."
869 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
870 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
871 for ((i = 0; i < 50; i++)); do
872 mkdir -p $DIR/$tdir/d${i}
873 touch $DIR/$tdir/f${i}
874 for ((j = 0; j < 50; j++)); do
875 touch $DIR/$tdir/d${i}/f${j}
879 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
880 [ "$STATUS" == "init" ] ||
881 error "(3) Expect 'init', but got '$STATUS'"
883 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN 0x160c
884 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160c
885 $START_NAMESPACE || error "(4) Fail to start LFSCK!"
888 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
889 [ "$STATUS" == "stopped" ] ||
890 error "(5) Expect 'stopped', but got '$STATUS'"
892 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
896 $START_NAMESPACE -s $BASE_SPEED1 || error "(6) Fail to start LFSCK!"
899 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
900 [ "$STATUS" == "scanning-phase2" ] ||
901 error "(7) Expect 'scanning-phase2', but got '$STATUS'"
903 local SPEED=$($SHOW_NAMESPACE |
904 awk '/^average_speed_phase2/ { print $2 }')
905 # There may be time error, normally it should be less than 2 seconds.
906 # We allow another 20% schedule error.
908 # MAX_MARGIN = 1.2 = 12 / 10
909 local MAX_SPEED=$((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) / \
910 RUN_TIME1 * 12 / 10))
911 [ $SPEED -lt $MAX_SPEED ] ||
912 error "(8) Got speed $SPEED, expected less than $MAX_SPEED"
915 local BASE_SPEED2=150
917 do_facet $SINGLEMDS \
918 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit $BASE_SPEED2
921 SPEED=$($SHOW_NAMESPACE | awk '/^average_speed_phase2/ { print $2 }')
922 # MIN_MARGIN = 0.8 = 8 / 10
923 local MIN_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 - TIME_DIFF) + \
924 BASE_SPEED2 * (RUN_TIME2 - TIME_DIFF)) / \
925 (RUN_TIME1 + RUN_TIME2) * 8 / 10))
926 [ $SPEED -gt $MIN_SPEED ] ||
927 error "(9) Got speed $SPEED, expected more than $MIN_SPEED"
929 # MAX_MARGIN = 1.2 = 12 / 10
930 MAX_SPEED=$(((BASE_SPEED1 * (RUN_TIME1 + TIME_DIFF) + \
931 BASE_SPEED2 * (RUN_TIME2 + TIME_DIFF)) / \
932 (RUN_TIME1 + RUN_TIME2) * 12 / 10))
933 [ $SPEED -lt $MAX_SPEED ] ||
934 error "(10) Got speed $SPEED, expected less than $MAX_SPEED"
936 do_facet $SINGLEMDS \
937 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
939 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
940 [ "$STATUS" == "completed" ] ||
941 error "(11) Expect 'completed', but got '$STATUS'"
943 run_test 9b "LFSCK speed control (2)"
947 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
948 skip "lookup(..)/linkea on ZFS issue" && return
951 echo "start $SINGLEMDS"
952 start $SINGLEMDS $MDT_DEVNAME $MOUNT_OPTS_SCRUB > /dev/null ||
953 error "(1) Fail to start MDS!"
955 mount_client $MOUNT || error "(2) Fail to start client!"
957 #define OBD_FAIL_LFSCK_LINKEA_CRASH 0x1603
958 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1603
959 for ((i = 0; i < 1000; i = $((i+2)))); do
960 mkdir -p $DIR/$tdir/d${i}
961 touch $DIR/$tdir/f${i}
962 for ((j = 0; j < 5; j++)); do
963 touch $DIR/$tdir/d${i}/f${j}
967 #define OBD_FAIL_LFSCK_LINKEA_MORE 0x1604
968 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1604
969 for ((i = 1; i < 1000; i = $((i+2)))); do
970 mkdir -p $DIR/$tdir/d${i}
971 touch $DIR/$tdir/f${i}
972 for ((j = 0; j < 5; j++)); do
973 touch $DIR/$tdir/d${i}/f${j}
977 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
978 ln $DIR/$tdir/f200 $DIR/$tdir/d200/dummy
981 mount_client $MOUNT || error "(3) Fail to start client!"
983 local STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
984 [ "$STATUS" == "init" ] ||
985 error "(4) Expect 'init', but got '$STATUS'"
987 $START_NAMESPACE -s 100 || error "(5) Fail to start LFSCK!"
990 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
991 [ "$STATUS" == "scanning-phase1" ] ||
992 error "(6) Expect 'scanning-phase1', but got '$STATUS'"
994 ls -ailR $MOUNT > /dev/null || error "(7) Fail to ls!"
996 touch $DIR/$tdir/d198/a0 || error "(8) Fail to touch!"
998 mkdir $DIR/$tdir/d199/a1 || error "(9) Fail to mkdir!"
1000 unlink $DIR/$tdir/f200 || error "(10) Fail to unlink!"
1002 rm -rf $DIR/$tdir/d201 || error "(11) Fail to rmdir!"
1004 mv $DIR/$tdir/f202 $DIR/$tdir/d203/ || error "(12) Fail to rename!"
1006 ln $DIR/$tdir/f204 $DIR/$tdir/d205/a3 || error "(13) Fail to hardlink!"
1008 ln -s $DIR/$tdir/d206 $DIR/$tdir/d207/a4 ||
1009 error "(14) Fail to softlink!"
1011 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1012 [ "$STATUS" == "scanning-phase1" ] ||
1013 error "(15) Expect 'scanning-phase1', but got '$STATUS'"
1015 do_facet $SINGLEMDS \
1016 $LCTL set_param -n mdd.${MDT_DEV}.lfsck_speed_limit 0
1017 umount_client $MOUNT
1019 STATUS=$($SHOW_NAMESPACE | awk '/^status/ { print $2 }')
1020 [ "$STATUS" == "completed" ] ||
1021 error "(16) Expect 'completed', but got '$STATUS'"
1023 run_test 10 "System is available during LFSCK scanning"
1026 ost_remove_lastid() {
1029 local rcmd="do_facet ost${ost}"
1031 echo "remove LAST_ID on ost${ost}: idx=${idx}"
1033 # step 1: local mount
1034 mount_fstype ost${ost} || return 1
1035 # step 2: remove the specified LAST_ID
1036 ${rcmd} rm -fv $(facet_mntpt ost${ost})/O/${idx}/{LAST_ID,d0/0}
1038 unmount_fstype ost${ost} || return 2
1045 formatall > /dev/null
1047 setupall > /dev/null
1050 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1051 createmany -o $DIR/$tdir/f 64
1056 ost_remove_lastid 1 0 || error "(1) Fail to remove LAST_ID"
1059 start ost1 $(ostdevname 1) $MOUNT_OPTS_NOSCRUB > /dev/null ||
1060 error "(2) Fail to start ost1"
1062 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1063 [ "$STATUS" == "init" ] ||
1064 error "(3) Expect 'init', but got '$STATUS'"
1066 #define OBD_FAIL_LFSCK_DELAY4 0x160e
1067 do_facet ost1 $LCTL set_param fail_val=3
1068 do_facet ost1 $LCTL set_param fail_loc=0x160e
1070 echo "trigger LFSCK for layout on ost1 to rebuild the LAST_ID(s)"
1071 $START_LAYOUT_ON_OST || error "(4) Fail to start LFSCK on OST!"
1073 wait_update_facet ost1 "$LCTL get_param -n \
1074 obdfilter.${OST_DEV}.lfsck_layout |
1075 awk '/^flags/ { print \\\$2 }'" "crashed_lastid" 60 || {
1080 do_facet ost1 $LCTL set_param fail_val=0
1081 do_facet ost1 $LCTL set_param fail_loc=0
1083 wait_update_facet ost1 "$LCTL get_param -n \
1084 obdfilter.${OST_DEV}.lfsck_layout |
1085 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1090 echo "the LAST_ID(s) should have been rebuilt"
1091 FLAGS=$($SHOW_LAYOUT_ON_OST | awk '/^flags/ { print $2 }')
1092 [ -z "$FLAGS" ] || error "(7) Expect empty flags, but got '$FLAGS'"
1094 run_test 11a "LFSCK can rebuild lost last_id"
1100 formatall > /dev/null
1102 setupall > /dev/null
1105 $SETSTRIPE -c 1 -i 0 $DIR/$tdir
1107 echo "set fail_loc=0x160d to skip the updating LAST_ID on-disk"
1108 #define OBD_FAIL_LFSCK_SKIP_LASTID 0x160d
1109 do_facet ost1 $LCTL set_param fail_loc=0x160d
1110 createmany -o $DIR/$tdir/f 64
1111 local lastid1=$(do_facet ost1 "lctl get_param -n \
1112 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1113 awk -F: '{ print $2 }')
1115 umount_client $MOUNT
1117 stop ost1 || error "(1) Fail to stop ost1"
1119 #define OBD_FAIL_OST_ENOSPC 0x215
1120 do_facet ost1 $LCTL set_param fail_loc=0x215
1123 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1124 error "(2) Fail to start ost1"
1126 local STATUS=$($SHOW_LAYOUT_ON_OST | awk '/^status/ { print $2 }')
1127 [ "$STATUS" == "init" ] ||
1128 error "(3) Expect 'init', but got '$STATUS'"
1130 for ((i = 0; i < 60; i++)); do
1131 lastid2=$(do_facet ost1 "lctl get_param -n \
1132 obdfilter.${ost1_svc}.last_id" | grep 0x100000000 |
1133 awk -F: '{ print $2 }')
1134 [ ! -z $lastid2 ] && break;
1138 echo "the on-disk LAST_ID should be smaller than the expected one"
1139 [ $lastid1 -gt $lastid2 ] ||
1140 error "(4) expect lastid1 [ $lastid1 ] > lastid2 [ $lastid2 ]"
1142 echo "trigger LFSCK for layout on ost1 to rebuild the on-disk LAST_ID"
1143 $START_LAYOUT_ON_OST || error "(5) Fail to start LFSCK on OST!"
1145 wait_update_facet ost1 "$LCTL get_param -n \
1146 obdfilter.${OST_DEV}.lfsck_layout |
1147 awk '/^status/ { print \\\$2 }'" "completed" 3 || {
1153 stop ost1 || error "(7) Fail to stop ost1"
1156 start ost1 $(ostdevname 1) $OST_MOUNT_OPTS ||
1157 error "(8) Fail to start ost1"
1159 echo "the on-disk LAST_ID should have been rebuilt"
1160 wait_update_facet ost1 "$LCTL get_param -n \
1161 obdfilter.${ost1_svc}.last_id | grep 0x100000000 |
1162 awk -F: '{ print \\\$2 }'" "$lastid1" 60 || {
1163 $LCTL get_param -n obdfilter.${ost1_svc}.last_id
1164 error "(9) expect lastid1 0x100000000:$lastid1"
1167 do_facet ost1 $LCTL set_param fail_loc=0
1169 run_test 11b "LFSCK can rebuild crashed last_id"
1172 [ $MDSCOUNT -lt 2 ] &&
1173 skip "We need at least 2 MDSes for test_12" && exit 0
1178 formatall > /dev/null
1180 setupall > /dev/null
1184 echo "All the LFSCK targets should be in 'init' status."
1185 for k in $(seq $MDSCOUNT); do
1186 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1187 mdd.$(facet_svc mds${k}).lfsck_layout |
1188 awk '/^status/ { print $2 }')
1189 [ "$STATUS" == "init" ] ||
1190 error "(1) MDS${k} Expect 'init', but got '$STATUS'"
1192 $LFS mkdir -i $((k - 1)) $DIR/$tdir/${k}
1193 createmany -o $DIR/$tdir/${k}/f 100
1196 echo "Start namespace LFSCK on all targets by single command (-s 1)."
1197 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1198 -s 1 || error "(2) Fail to start LFSCK on all devices!"
1200 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1201 for k in $(seq $MDSCOUNT); do
1202 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1203 mdd.$(facet_svc mds${k}).lfsck_namespace |
1204 awk '/^status/ { print $2 }')
1205 [ "$STATUS" == "scanning-phase1" ] ||
1206 error "(3) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1209 echo "Stop namespace LFSCK on all targets by single lctl command."
1210 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1211 error "(4) Fail to stop LFSCK on all devices!"
1213 echo "All the LFSCK targets should be in 'stopped' status."
1214 for k in $(seq $MDSCOUNT); do
1215 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1216 mdd.$(facet_svc mds${k}).lfsck_namespace |
1217 awk '/^status/ { print $2 }')
1218 [ "$STATUS" == "stopped" ] ||
1219 error "(5) MDS${k} Expect 'stopped', but got '$STATUS'"
1222 echo "Re-start namespace LFSCK on all targets by single command (-s 0)."
1223 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t namespace -A \
1224 -s 0 -r || error "(6) Fail to start LFSCK on all devices!"
1226 echo "All the LFSCK targets should be in 'completed' status."
1227 for k in $(seq $MDSCOUNT); do
1228 wait_update_facet mds${k} "$LCTL get_param -n \
1229 mdd.$(facet_svc mds${k}).lfsck_namespace |
1230 awk '/^status/ { print \\\$2 }'" "completed" 8 ||
1231 error "(7) MDS${k} is not the expected 'completed'"
1234 echo "Start layout LFSCK on all targets by single command (-s 1)."
1235 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1236 -s 1 || error "(8) Fail to start LFSCK on all devices!"
1238 echo "All the LFSCK targets should be in 'scanning-phase1' status."
1239 for k in $(seq $MDSCOUNT); do
1240 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1241 mdd.$(facet_svc mds${k}).lfsck_layout |
1242 awk '/^status/ { print $2 }')
1243 [ "$STATUS" == "scanning-phase1" ] ||
1244 error "(9) MDS${k} Expect 'scanning-phase1', but got '$STATUS'"
1247 echo "Stop layout LFSCK on all targets by single lctl command."
1248 do_facet mds1 $LCTL lfsck_stop -M ${FSNAME}-MDT0000 -A ||
1249 error "(10) Fail to stop LFSCK on all devices!"
1251 echo "All the LFSCK targets should be in 'stopped' status."
1252 for k in $(seq $MDSCOUNT); do
1253 local STATUS=$(do_facet mds${k} $LCTL get_param -n \
1254 mdd.$(facet_svc mds${k}).lfsck_layout |
1255 awk '/^status/ { print $2 }')
1256 [ "$STATUS" == "stopped" ] ||
1257 error "(11) MDS${k} Expect 'stopped', but got '$STATUS'"
1260 for k in $(seq $OSTCOUNT); do
1261 local STATUS=$(do_facet ost${k} $LCTL get_param -n \
1262 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1263 awk '/^status/ { print $2 }')
1264 [ "$STATUS" == "stopped" ] ||
1265 error "(12) OST${k} Expect 'stopped', but got '$STATUS'"
1268 echo "Re-start layout LFSCK on all targets by single command (-s 0)."
1269 do_facet mds1 $LCTL lfsck_start -M ${FSNAME}-MDT0000 -t layout -A \
1270 -s 0 -r || error "(13) Fail to start LFSCK on all devices!"
1272 echo "All the LFSCK targets should be in 'completed' status."
1273 for k in $(seq $MDSCOUNT); do
1274 # The LFSCK status query internal is 30 seconds. For the case
1275 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1276 # time to guarantee the status sync up.
1277 wait_update_facet mds${k} "$LCTL get_param -n \
1278 mdd.$(facet_svc mds${k}).lfsck_layout |
1279 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1280 error "(14) MDS${k} is not the expected 'completed'"
1283 run_test 12 "single command to trigger LFSCK on all devices"
1287 echo "The lmm_oi in layout EA should be consistent with the MDT-object"
1288 echo "FID; otherwise, the LFSCK should re-generate the lmm_oi from the"
1289 echo "MDT-object FID."
1295 formatall > /dev/null
1297 setupall > /dev/null
1301 echo "Inject failure stub to simulate bad lmm_oi"
1302 #define OBD_FAIL_LFSCK_BAD_LMMOI 0x160f
1303 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x160f
1304 createmany -o $DIR/$tdir/f 32
1305 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1307 echo "stopall to cleanup object cache"
1310 setupall > /dev/null
1312 echo "Trigger layout LFSCK to find out the bad lmm_oi and fix them"
1313 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1315 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1316 mdd.${MDT_DEV}.lfsck_layout |
1317 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1319 local repaired=$($SHOW_LAYOUT |
1320 awk '/^repaired_others/ { print $2 }')
1321 [ $repaired -eq 32 ] ||
1322 error "(3) Fail to repair crashed lmm_oi: $repaired"
1324 run_test 13 "LFSCK can repair crashed lmm_oi"
1328 echo "The OST-object referenced by the MDT-object should be there;"
1329 echo "otherwise, the LFSCK should re-create the missed OST-object."
1335 formatall > /dev/null
1337 setupall > /dev/null
1340 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1342 echo "Inject failure stub to simulate dangling referenced MDT-object"
1343 #define OBD_FAIL_LFSCK_DANGLING 0x1610
1344 do_facet ost1 $LCTL set_param fail_loc=0x1610
1345 createmany -o $DIR/$tdir/f 64
1346 do_facet ost1 $LCTL set_param fail_loc=0
1348 echo "stopall to cleanup object cache"
1351 setupall > /dev/null
1353 echo "'ls' should fail because of dangling referenced MDT-object"
1354 ls -ail $DIR/$tdir > /dev/null 2>&1 && error "(1) ls should fail."
1356 echo "Trigger layout LFSCK to find out dangling reference and fix them"
1357 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1359 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1360 mdd.${MDT_DEV}.lfsck_layout |
1361 awk '/^status/ { print \\\$2 }'" "completed" 6 || return 3
1363 local repaired=$($SHOW_LAYOUT |
1364 awk '/^repaired_dangling/ { print $2 }')
1365 [ $repaired -eq 32 ] ||
1366 error "(4) Fail to repair dangling reference: $repaired"
1368 echo "'ls' should success after layout LFSCK repairing"
1369 ls -ail $DIR/$tdir > /dev/null || error "(5) ls should success."
1371 run_test 14 "LFSCK can repair MDT-object with dangling reference"
1375 echo "If the OST-object referenced by the MDT-object back points"
1376 echo "to some non-exist MDT-object, then the LFSCK should repair"
1377 echo "the OST-object to back point to the right MDT-object."
1383 formatall > /dev/null
1385 setupall > /dev/null
1388 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1390 echo "Inject failure stub to make the OST-object to back point to"
1391 echo "non-exist MDT-object."
1392 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
1394 do_facet ost1 $LCTL set_param fail_loc=0x1611
1395 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1396 cancel_lru_locks osc
1399 do_facet ost1 $LCTL set_param fail_loc=0
1401 echo "stopall to cleanup object cache"
1404 setupall > /dev/null
1406 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1407 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1409 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1410 mdd.${MDT_DEV}.lfsck_layout |
1411 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1413 local repaired=$($SHOW_LAYOUT |
1414 awk '/^repaired_unmatched_pair/ { print $2 }')
1415 [ $repaired -eq 1 ] ||
1416 error "(3) Fail to repair unmatched pair: $repaired"
1418 run_test 15a "LFSCK can repair unmatched MDT-object/OST-object pairs (1)"
1422 echo "If the OST-object referenced by the MDT-object back points"
1423 echo "to other MDT-object that doesn't recognize the OST-object,"
1424 echo "then the LFSCK should repair it to back point to the right"
1425 echo "MDT-object (the first one)."
1431 formatall > /dev/null
1433 setupall > /dev/null
1436 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1437 dd if=/dev/zero of=$DIR/$tdir/guard bs=1K count=1
1438 cancel_lru_locks osc
1440 echo "Inject failure stub to make the OST-object to back point to"
1441 echo "other MDT-object"
1443 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR2 0x1612
1444 do_facet ost1 $LCTL set_param fail_loc=0x1612
1445 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1446 cancel_lru_locks osc
1449 do_facet ost1 $LCTL set_param fail_loc=0
1451 echo "stopall to cleanup object cache"
1454 setupall > /dev/null
1456 echo "Trigger layout LFSCK to find out unmatched pairs and fix them"
1457 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1459 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1460 mdd.${MDT_DEV}.lfsck_layout |
1461 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1463 local repaired=$($SHOW_LAYOUT |
1464 awk '/^repaired_unmatched_pair/ { print $2 }')
1465 [ $repaired -eq 1 ] ||
1466 error "(3) Fail to repair unmatched pair: $repaired"
1468 run_test 15b "LFSCK can repair unmatched MDT-object/OST-object pairs (2)"
1472 echo "If the OST-object's owner information does not match the owner"
1473 echo "information stored in the MDT-object, then the LFSCK trust the"
1474 echo "MDT-object and update the OST-object's owner information."
1480 formatall > /dev/null
1482 setupall > /dev/null
1485 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1486 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=1
1487 cancel_lru_locks osc
1491 echo "Inject failure stub to skip OST-object owner changing"
1492 #define OBD_FAIL_LFSCK_BAD_OWNER 0x1613
1493 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1613
1494 chown 1.1 $DIR/$tdir/f0
1495 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1497 echo "Trigger layout LFSCK to find out inconsistent OST-object owner"
1500 $START_LAYOUT || error "(1) Fail to start LFSCK for layout!"
1502 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1503 mdd.${MDT_DEV}.lfsck_layout |
1504 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 2
1506 local repaired=$($SHOW_LAYOUT |
1507 awk '/^repaired_inconsistent_owner/ { print $2 }')
1508 [ $repaired -eq 1 ] ||
1509 error "(3) Fail to repair inconsistent owner: $repaired"
1511 run_test 16 "LFSCK can repair inconsistent MDT-object/OST-object owner"
1515 echo "If more than one MDT-objects reference the same OST-object,"
1516 echo "and the OST-object only recognizes one MDT-object, then the"
1517 echo "LFSCK should create new OST-objects for such non-recognized"
1524 formatall > /dev/null
1526 setupall > /dev/null
1529 $LFS setstripe -c 1 -i 0 $DIR/$tdir
1531 echo "Inject failure stub to make two MDT-objects to refernce"
1532 echo "the OST-object"
1534 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1535 #define OBD_FAIL_LFSCK_MULTIPLE_REF 0x1614
1536 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1614
1538 dd if=/dev/zero of=$DIR/$tdir/guard bs=1M count=1
1539 cancel_lru_locks osc
1543 createmany -o $DIR/$tdir/f 1 > /dev/null 2>&1
1545 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1546 do_facet $SINGLEMDS $LCTL set_param fail_val=0
1548 echo "stopall to cleanup object cache"
1551 setupall > /dev/null
1553 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard use the same OST-objects"
1554 local size=$(ls -l $DIR/$tdir/f0 | awk '{ print $5 }')
1555 [ $size -eq 1048576 ] ||
1556 error "(1) f0 (wrong) size should be 1048576, but got $size"
1558 echo "Trigger layout LFSCK to find out multiple refenced MDT-objects"
1561 $START_LAYOUT || error "(2) Fail to start LFSCK for layout!"
1563 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
1564 mdd.${MDT_DEV}.lfsck_layout |
1565 awk '/^status/ { print \\\$2 }'" "completed" 3 || return 3
1567 local repaired=$($SHOW_LAYOUT |
1568 awk '/^repaired_multiple_referenced/ { print $2 }')
1569 [ $repaired -eq 1 ] ||
1570 error "(4) Fail to repair multiple references: $repaired"
1572 echo "$DIR/$tdir/f0 and $DIR/$tdir/guard should use diff OST-objects"
1573 dd if=/dev/zero of=$DIR/$tdir/f0 bs=1M count=2 ||
1574 error "(5) Fail to write f0."
1575 size=$(ls -l $DIR/$tdir/guard | awk '{ print $5 }')
1576 [ $size -eq 1048576 ] ||
1577 error "(6) guard size should be 1048576, but got $size"
1579 run_test 17 "LFSCK can repair multiple references"
1582 [ $MDSCOUNT -lt 2 ] &&
1583 skip "We need at least 2 MDSes for test_18a" && exit 0
1585 [ $OSTCOUNT -lt 2 ] &&
1586 skip "We need at least 2 OSTs for test_18a" && exit 0
1589 echo "The target MDT-object is there, but related stripe information"
1590 echo "is lost or partly lost. The LFSCK should regenerate the missed"
1591 echo "layout EA entries."
1597 formatall > /dev/null
1599 setupall > /dev/null
1602 $LFS mkdir -i 0 $DIR/$tdir/a1
1603 $LFS mkdir -i 1 $DIR/$tdir/a2
1604 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1605 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1606 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1607 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1609 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1611 $LFS path2fid $DIR/$tdir/a1/f1
1612 $LFS getstripe $DIR/$tdir/a1/f1
1613 $LFS path2fid $DIR/$tdir/a2/f2
1614 $LFS getstripe $DIR/$tdir/a2/f2
1616 cancel_lru_locks osc
1618 echo "Inject failure, to make the MDT-object lost its layout EA"
1619 #define OBD_FAIL_LFSCK_LOST_STRIPE 0x1615
1620 do_facet mds1 $LCTL set_param fail_loc=0x1615
1621 chown 1.1 $DIR/$tdir/a1/f1
1622 do_facet mds2 $LCTL set_param fail_loc=0x1615
1623 chown 1.1 $DIR/$tdir/a2/f2
1626 do_facet mds1 $LCTL set_param fail_loc=0
1627 do_facet mds2 $LCTL set_param fail_loc=0
1629 echo "stopall to cleanup object cache"
1632 setupall > /dev/null
1634 echo "The file size should be incorrect since layout EA is lost"
1635 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1636 [ "$cur_size" != "$saved_size" ] ||
1637 error "(1) Expect incorrect file1 size"
1639 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1640 [ "$cur_size" != "$saved_size" ] ||
1641 error "(2) Expect incorrect file2 size"
1643 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1644 $START_LAYOUT -o || error "(3) Fail to start LFSCK for layout!"
1646 for k in $(seq $MDSCOUNT); do
1647 # The LFSCK status query internal is 30 seconds. For the case
1648 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1649 # time to guarantee the status sync up.
1650 wait_update_facet mds${k} "$LCTL get_param -n \
1651 mdd.$(facet_svc mds${k}).lfsck_layout |
1652 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1653 error "(4) MDS${k} is not the expected 'completed'"
1656 for k in $(seq $OSTCOUNT); do
1657 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1658 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1659 awk '/^status/ { print $2 }')
1660 [ "$cur_status" == "completed" ] ||
1661 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
1665 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1666 mdd.$(facet_svc mds${k}).lfsck_layout |
1667 awk '/^repaired_orphan/ { print $2 }')
1668 [ $repaired -eq ${k} ] ||
1669 error "(6) Expect ${k} fixed on mds${k}, but got: $repaired"
1672 $LFS path2fid $DIR/$tdir/a1/f1
1673 $LFS getstripe $DIR/$tdir/a1/f1
1674 $LFS path2fid $DIR/$tdir/a2/f2
1675 $LFS getstripe $DIR/$tdir/a2/f2
1677 echo "The file size should be correct after layout LFSCK scanning"
1678 cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1679 [ "$cur_size" == "$saved_size" ] ||
1680 error "(7) Expect file1 size $saved_size, but got $cur_size"
1682 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1683 [ "$cur_size" == "$saved_size" ] ||
1684 error "(8) Expect file2 size $saved_size, but got $cur_size"
1686 run_test 18a "Find out orphan OST-object and repair it (1)"
1689 [ $MDSCOUNT -lt 2 ] &&
1690 skip "We need at least 2 MDSes for test_18b" && exit 0
1692 [ $OSTCOUNT -lt 2 ] &&
1693 skip "We need at least 2 OSTs for test_18b" && exit 0
1696 echo "The target MDT-object is lost. The LFSCK should re-create the"
1697 echo "MDT-object under .lustre/lost+found/MDTxxxx. The admin should"
1698 echo "can move it back to normal namespace manually."
1704 formatall > /dev/null
1706 setupall > /dev/null
1709 $LFS mkdir -i 0 $DIR/$tdir/a1
1710 $LFS mkdir -i 1 $DIR/$tdir/a2
1711 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1712 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1713 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1714 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1715 local saved_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1716 local fid1=$($LFS path2fid $DIR/$tdir/a1/f1)
1718 $LFS getstripe $DIR/$tdir/a1/f1
1719 local fid2=$($LFS path2fid $DIR/$tdir/a2/f2)
1721 $LFS getstripe $DIR/$tdir/a2/f2
1723 cancel_lru_locks osc
1725 echo "Inject failure, to simulate the case of missing the MDT-object"
1726 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1727 do_facet mds1 $LCTL set_param fail_loc=0x1616
1728 rm -f $DIR/$tdir/a1/f1
1729 do_facet mds2 $LCTL set_param fail_loc=0x1616
1730 rm -f $DIR/$tdir/a2/f2
1733 do_facet mds1 $LCTL set_param fail_loc=0
1734 do_facet mds2 $LCTL set_param fail_loc=0
1736 echo "stopall to cleanup object cache"
1739 setupall > /dev/null
1741 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1742 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1744 for k in $(seq $MDSCOUNT); do
1745 # The LFSCK status query internal is 30 seconds. For the case
1746 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1747 # time to guarantee the status sync up.
1748 wait_update_facet mds${k} "$LCTL get_param -n \
1749 mdd.$(facet_svc mds${k}).lfsck_layout |
1750 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1751 error "(2) MDS${k} is not the expected 'completed'"
1754 for k in $(seq $OSTCOUNT); do
1755 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1756 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1757 awk '/^status/ { print $2 }')
1758 [ "$cur_status" == "completed" ] ||
1759 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1763 local repaired=$(do_facet mds${k} $LCTL get_param -n \
1764 mdd.$(facet_svc mds${k}).lfsck_layout |
1765 awk '/^repaired_orphan/ { print $2 }')
1766 [ $repaired -eq ${k} ] ||
1767 error "(4) Expect ${k} fixed on mds${k}, but got: $repaired"
1770 echo "Move the files from ./lustre/lost+found/MDTxxxx to namespace"
1771 mv $MOUNT/.lustre/lost+found/MDT0000/R-${fid1} $DIR/$tdir/a1/f1 ||
1772 error "(5) Fail to move $MOUNT/.lustre/lost+found/MDT0000/R-${fid1}"
1774 mv $MOUNT/.lustre/lost+found/MDT0001/R-${fid2} $DIR/$tdir/a2/f2 ||
1775 error "(6) Fail to move $MOUNT/.lustre/lost+found/MDT0001/R-${fid2}"
1777 $LFS path2fid $DIR/$tdir/a1/f1
1778 $LFS getstripe $DIR/$tdir/a1/f1
1779 $LFS path2fid $DIR/$tdir/a2/f2
1780 $LFS getstripe $DIR/$tdir/a2/f2
1782 echo "The file size should be correct after layout LFSCK scanning"
1783 local cur_size=$(ls -il $DIR/$tdir/a1/f1 | awk '{ print $6 }')
1784 [ "$cur_size" == "$saved_size" ] ||
1785 error "(7) Expect file1 size $saved_size, but got $cur_size"
1787 cur_size=$(ls -il $DIR/$tdir/a2/f2 | awk '{ print $6 }')
1788 [ "$cur_size" == "$saved_size" ] ||
1789 error "(8) Expect file2 size $saved_size, but got $cur_size"
1791 run_test 18b "Find out orphan OST-object and repair it (2)"
1794 [ $MDSCOUNT -lt 2 ] &&
1795 skip "We need at least 2 MDSes for test_18c" && exit 0
1797 [ $OSTCOUNT -lt 2 ] &&
1798 skip "We need at least 2 OSTs for test_18c" && exit 0
1801 echo "The target MDT-object is lost, and the OST-object FID is missing."
1802 echo "The LFSCK should re-create the MDT-object with new FID under the "
1803 echo "directory .lustre/lost+found/MDTxxxx."
1809 formatall > /dev/null
1811 setupall > /dev/null
1814 $LFS mkdir -i 0 $DIR/$tdir/a1
1815 $LFS mkdir -i 1 $DIR/$tdir/a2
1816 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1817 $LFS setstripe -c 2 -i 1 -s 1M $DIR/$tdir/a2
1819 echo "Inject failure, to simulate the case of missing parent FID"
1820 #define OBD_FAIL_LFSCK_NOPFID 0x1617
1821 do_facet ost1 $LCTL set_param fail_loc=0x1617
1822 do_facet ost2 $LCTL set_param fail_loc=0x1617
1824 dd if=/dev/zero of=$DIR/$tdir/a1/f1 bs=1M count=2
1825 dd if=/dev/zero of=$DIR/$tdir/a2/f2 bs=1M count=2
1826 $LFS getstripe $DIR/$tdir/a1/f1
1827 $LFS getstripe $DIR/$tdir/a2/f2
1829 cancel_lru_locks osc
1831 echo "Inject failure, to simulate the case of missing the MDT-object"
1832 #define OBD_FAIL_LFSCK_LOST_MDTOBJ 0x1616
1833 do_facet mds1 $LCTL set_param fail_loc=0x1616
1834 rm -f $DIR/$tdir/a1/f1
1835 do_facet mds2 $LCTL set_param fail_loc=0x1616
1836 rm -f $DIR/$tdir/a2/f2
1839 do_facet mds1 $LCTL set_param fail_loc=0
1840 do_facet mds2 $LCTL set_param fail_loc=0
1842 echo "stopall to cleanup object cache"
1845 setupall > /dev/null
1847 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1848 $START_LAYOUT -o || error "(1) Fail to start LFSCK for layout!"
1850 for k in $(seq $MDSCOUNT); do
1851 # The LFSCK status query internal is 30 seconds. For the case
1852 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1853 # time to guarantee the status sync up.
1854 wait_update_facet mds${k} "$LCTL get_param -n \
1855 mdd.$(facet_svc mds${k}).lfsck_layout |
1856 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1857 error "(2) MDS${k} is not the expected 'completed'"
1860 for k in $(seq $OSTCOUNT); do
1861 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1862 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1863 awk '/^status/ { print $2 }')
1864 [ "$cur_status" == "completed" ] ||
1865 error "(3) OST${k} Expect 'completed', but got '$cur_status'"
1868 local repaired=$(do_facet mds1 $LCTL get_param -n \
1869 mdd.$(facet_svc mds1).lfsck_layout |
1870 awk '/^repaired_orphan/ { print $2 }')
1871 [ $repaired -eq 3 ] ||
1872 error "(4) Expect 3 fixed on mds1, but got: $repaired"
1874 repaired=$(do_facet mds2 $LCTL get_param -n \
1875 mdd.$(facet_svc mds2).lfsck_layout |
1876 awk '/^repaired_orphan/ { print $2 }')
1877 [ $repaired -eq 0 ] ||
1878 error "(5) Expect 0 fixed on mds2, but got: $repaired"
1880 echo "There should be some stub under .lustre/lost+found/MDT0001/"
1881 ls -ail $MOUNT/.lustre/lost+found/MDT0001/N-* &&
1882 error "(6) .lustre/lost+found/MDT0001/ should be empty"
1884 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1885 ls -ail $MOUNT/.lustre/lost+found/MDT0000/N-* ||
1886 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
1888 run_test 18c "Find out orphan OST-object and repair it (3)"
1892 echo "The target MDT-object layout EA slot is occpuied by some new"
1893 echo "created OST-object when repair dangling reference case. Such"
1894 echo "conflict OST-object has never been modified. Then when found"
1895 echo "the orphan OST-object, LFSCK will replace it with the orphan"
1902 formatall > /dev/null
1904 setupall > /dev/null
1906 mkdir -p $DIR/$tdir/a1
1907 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
1908 echo "guard" > $DIR/$tdir/a1/f1
1909 echo "foo" > $DIR/$tdir/a1/f2
1910 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1911 $LFS path2fid $DIR/$tdir/a1/f1
1912 $LFS getstripe $DIR/$tdir/a1/f1
1913 $LFS path2fid $DIR/$tdir/a1/f2
1914 $LFS getstripe $DIR/$tdir/a1/f2
1916 cancel_lru_locks osc
1918 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
1919 echo "to reference the same OST-object (which is f1's OST-obejct)."
1920 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
1921 echo "dangling reference case, but f2's old OST-object is there."
1924 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
1925 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
1926 chown 1.1 $DIR/$tdir/a1/f2
1927 rm -f $DIR/$tdir/a1/f1
1930 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
1932 echo "stopall to cleanup object cache"
1935 setupall > /dev/null
1937 echo "The file size should be incorrect since dangling referenced"
1938 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1939 [ "$cur_size" != "$saved_size" ] ||
1940 error "(1) Expect incorrect file2 size"
1942 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
1943 $START_LAYOUT -o || error "(2) Fail to start LFSCK for layout!"
1945 for k in $(seq $MDSCOUNT); do
1946 # The LFSCK status query internal is 30 seconds. For the case
1947 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
1948 # time to guarantee the status sync up.
1949 wait_update_facet mds${k} "$LCTL get_param -n \
1950 mdd.$(facet_svc mds${k}).lfsck_layout |
1951 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
1952 error "(3) MDS${k} is not the expected 'completed'"
1955 for k in $(seq $OSTCOUNT); do
1956 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
1957 obdfilter.$(facet_svc ost${k}).lfsck_layout |
1958 awk '/^status/ { print $2 }')
1959 [ "$cur_status" == "completed" ] ||
1960 error "(4) OST${k} Expect 'completed', but got '$cur_status'"
1963 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
1964 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
1965 awk '/^repaired_orphan/ { print $2 }')
1966 [ $repaired -eq 1 ] ||
1967 error "(5) Expect 1 orphan has been fixed, but got: $repaired"
1969 echo "The file size should be correct after layout LFSCK scanning"
1970 cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
1971 [ "$cur_size" == "$saved_size" ] ||
1972 error "(6) Expect file2 size $saved_size, but got $cur_size"
1974 echo "There should be some stub under .lustre/lost+found/MDT0000/"
1975 ls -ail $MOUNT/.lustre/lost+found/MDT0000/ &&
1976 error "(7) .lustre/lost+found/MDT0000/ should be empty"
1978 echo "The LFSCK should find back the original data."
1979 cat $DIR/$tdir/a1/f2
1980 $LFS path2fid $DIR/$tdir/a1/f2
1981 $LFS getstripe $DIR/$tdir/a1/f2
1983 run_test 18d "Find out orphan OST-object and repair it (4)"
1987 echo "The target MDT-object layout EA slot is occpuied by some new"
1988 echo "created OST-object when repair dangling reference case. Such"
1989 echo "conflict OST-object has been modified by others. To keep the"
1990 echo "new data, the LFSCK will create a new file to refernece this"
1991 echo "old orphan OST-object."
1997 formatall > /dev/null
1999 setupall > /dev/null
2001 mkdir -p $DIR/$tdir/a1
2002 $LFS setstripe -c 1 -i 0 -s 1M $DIR/$tdir/a1
2003 echo "guard" > $DIR/$tdir/a1/f1
2004 echo "foo" > $DIR/$tdir/a1/f2
2005 local saved_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2006 $LFS path2fid $DIR/$tdir/a1/f1
2007 $LFS getstripe $DIR/$tdir/a1/f1
2008 $LFS path2fid $DIR/$tdir/a1/f2
2009 $LFS getstripe $DIR/$tdir/a1/f2
2011 cancel_lru_locks osc
2013 echo "Inject failure to make $DIR/$tdir/a1/f1 and $DIR/$tdir/a1/f2"
2014 echo "to reference the same OST-object (which is f1's OST-obejct)."
2015 echo "Then drop $DIR/$tdir/a1/f1 and its OST-object, so f2 becomes"
2016 echo "dangling reference case, but f2's old OST-object is there."
2019 #define OBD_FAIL_LFSCK_CHANGE_STRIPE 0x1618
2020 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1618
2021 chown 1.1 $DIR/$tdir/a1/f2
2022 rm -f $DIR/$tdir/a1/f1
2025 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2027 echo "stopall to cleanup object cache"
2030 setupall > /dev/null
2032 echo "The file size should be incorrect since dangling referenced"
2033 local cur_size=$(ls -il $DIR/$tdir/a1/f2 | awk '{ print $6 }')
2034 [ "$cur_size" != "$saved_size" ] ||
2035 error "(1) Expect incorrect file2 size"
2037 #define OBD_FAIL_LFSCK_DELAY3 0x1602
2038 do_facet $SINGLEMDS $LCTL set_param fail_val=10
2039 do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1602
2041 echo "Trigger layout LFSCK on all devices to find out orphan OST-object"
2042 $START_LAYOUT -o || error "(2) Fail to start LFSCK for layout!"
2044 wait_update_facet mds1 "$LCTL get_param -n \
2045 mdd.$(facet_svc mds1).lfsck_layout |
2046 awk '/^status/ { print \\\$2 }'" "scanning-phase2" 6 ||
2047 error "(3) MDS1 is not the expected 'scanning-phase2'"
2049 # to guarantee all updates are synced.
2053 echo "Write new data to f2 to modify the new created OST-object."
2054 echo "dummy" >> $DIR/$tdir/a1/f2
2056 do_facet $SINGLEMDS $LCTL set_param fail_val=0
2057 do_facet $SINGLEMDS $LCTL set_param fail_loc=0
2059 for k in $(seq $MDSCOUNT); do
2060 # The LFSCK status query internal is 30 seconds. For the case
2061 # of some LFSCK_NOTIFY RPCs failure/lost, we will wait enough
2062 # time to guarantee the status sync up.
2063 wait_update_facet mds${k} "$LCTL get_param -n \
2064 mdd.$(facet_svc mds${k}).lfsck_layout |
2065 awk '/^status/ { print \\\$2 }'" "completed" 32 ||
2066 error "(4) MDS${k} is not the expected 'completed'"
2069 for k in $(seq $OSTCOUNT); do
2070 local cur_status=$(do_facet ost${k} $LCTL get_param -n \
2071 obdfilter.$(facet_svc ost${k}).lfsck_layout |
2072 awk '/^status/ { print $2 }')
2073 [ "$cur_status" == "completed" ] ||
2074 error "(5) OST${k} Expect 'completed', but got '$cur_status'"
2077 local repaired=$(do_facet $SINGLEMDS $LCTL get_param -n \
2078 mdd.$(facet_svc $SINGLEMDS).lfsck_layout |
2079 awk '/^repaired_orphan/ { print $2 }')
2080 [ $repaired -eq 1 ] ||
2081 error "(6) Expect 1 orphan has been fixed, but got: $repaired"
2083 echo "There should be stub file under .lustre/lost+found/MDT0000/"
2084 local cname=$(ls $MOUNT/.lustre/lost+found/MDT0000/C-*)
2086 error "(7) .lustre/lost+found/MDT0000/ should not be empty"
2088 echo "The stub file should keep the original f2 data"
2089 cur_size=$(ls -il $cname | awk '{ print $6 }')
2090 [ "$cur_size" == "$saved_size" ] ||
2091 error "(8) Expect file2 size $saved_size, but got $cur_size"
2094 $LFS path2fid $cname
2095 $LFS getstripe $cname
2097 echo "The f2 should contains new data."
2098 cat $DIR/$tdir/a1/f2
2099 $LFS path2fid $DIR/$tdir/a1/f2
2100 $LFS getstripe $DIR/$tdir/a1/f2
2102 run_test 18e "Find out orphan OST-object and repair it (5)"
2108 formatall > /dev/null
2110 setupall > /dev/null
2113 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2115 echo "foo" > $DIR/$tdir/a0
2116 echo "guard" > $DIR/$tdir/a1
2118 cancel_lru_locks osc
2119 umount_client $MOUNT || error "(1) Fail to stop client!"
2120 mount_client $MOUNT || error "(2) Fail to start client!"
2122 echo "Inject failure, then client will offer wrong parent FID when read"
2123 do_facet ost1 $LCTL set_param -n \
2124 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2125 #define OBD_FAIL_LFSCK_INVALID_PFID 0x1619
2126 $LCTL set_param fail_loc=0x1619
2128 echo "Read RPC with wrong parent FID should be denied"
2129 cat $DIR/$tdir/a0 && error "(3) Read should be denied!"
2130 $LCTL set_param fail_loc=0
2132 run_test 19a "OST-object inconsistency self detect"
2138 formatall > /dev/null
2140 setupall > /dev/null
2143 $LFS setstripe -c 1 -i 0 $DIR/$tdir
2145 echo "Inject failure stub to make the OST-object to back point to"
2146 echo "non-exist MDT-object"
2148 #define OBD_FAIL_LFSCK_UNMATCHED_PAIR1 0x1611
2149 do_facet ost1 $LCTL set_param fail_loc=0x1611
2150 echo "foo" > $DIR/$tdir/f0
2151 cancel_lru_locks osc
2154 do_facet ost1 $LCTL set_param fail_loc=0
2156 echo "Nothing should be fixed since self detect and repair is disabled"
2157 local repaired=$(do_facet ost1 $LCTL get_param -n \
2158 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2159 awk '/^repaired/ { print $2 }')
2160 [ $repaired -eq 0 ] ||
2161 error "(1) Expected 0 repaired, but got $repaired"
2163 echo "Read RPC with right parent FID should be accepted,"
2164 echo "and cause parent FID on OST to be fixed"
2166 do_facet ost1 $LCTL set_param -n \
2167 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid 1
2168 cat $DIR/$tdir/f0 || error "(2) Read should not be denied!"
2170 repaired=$(do_facet ost1 $LCTL get_param -n \
2171 obdfilter.${FSNAME}-OST0000.lfsck_verify_pfid |
2172 awk '/^repaired/ { print $2 }')
2173 [ $repaired -eq 1 ] ||
2174 error "(3) Expected 1 repaired, but got $repaired"
2176 run_test 19b "OST-object inconsistency self repair"
2178 $LCTL set_param debug=-lfsck > /dev/null || true
2180 # restore MDS/OST size
2181 MDSSIZE=${SAVED_MDSSIZE}
2182 OSTSIZE=${SAVED_OSTSIZE}
2184 # cleanup the system at last