7 LUSTRE=${LUSTRE:-$(dirname $0)/..}
8 . $LUSTRE/tests/test-framework.sh
12 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
15 require_dsh_mds || exit 0
16 require_dsh_ost || exit 0
19 skip "skip lfsck performance test under non-SLOW mode"
21 NTHREADS=${NTHREADS:-0}
23 MINCOUNT=${MINCOUNT:-4096}
24 MAXCOUNT=${MAXCOUNT:-8192}
25 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
26 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
27 BASE_COUNT=${BASE_COUNT:-8192}
29 INCFACTOR=${INCFACTOR:-25} #percent
30 MINSUBDIR=${MINSUBDIR:-1}
31 MAXSUBDIR=${MAXSUBDIR:-2}
32 TOTSUBDIR=${TOTSUBDIR:-2}
35 RCMD="do_facet ${SINGLEMDS}"
36 RLCTL="${RCMD} ${LCTL}"
37 MDT_DEV="${FSNAME}-MDT0000"
38 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
39 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
40 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
41 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
42 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
43 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
45 if [ ${NTHREADS} -eq 0 ]; then
46 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
47 NTHREADS=$((CPUCORE * 2))
51 ${RCMD} "modprobe obdecho"
53 ${ECHOCMD} "${LCTL} <<-EOF
54 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
60 ${ECHOCMD} "${LCTL} <<-EOF
61 --device lfsck-MDT0000 cleanup
62 --device lfsck-MDT0000 detach
67 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
70 ${ECHOCMD} "${LCTL} --device ${echodev} test_mkdir ${tdir}"
72 for ((j = 1; j < ${threads}; j++)); do
73 ${ECHOCMD} "${LCTL} --device ${echodev} test_mkdir ${tdir}${j}"
76 ${ECHOCMD} "${LCTL} --device ${echodev} \
77 --threads ${threads} 0 ${echodev} \
78 test_create -d${tdir} -D${threads} -b${lbase} -c0 -n${usize}"
82 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
83 REFORMAT="yes" cleanup_and_setup_lustre
86 lfsck_create_nfiles() {
91 local ldir="/test-${lbase}"
96 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
97 local usize=$((count / NTHREADS))
98 [ ${usize} -eq 0 ] && break
99 local tdir=${ldir}-${cycle}-
101 echo "[cycle: ${cycle}] [threads: ${threads}]"\
102 "[files: ${count}] [basedir: ${tdir}]"
103 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
104 error "Fail to start MDS!"
105 #define OBD_FAIL_FID_IGIF 0x1504
106 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
112 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
113 stop ${SINGLEMDS} || error "Fail to stop MDS!"
115 total=$((total - usize * NTHREADS))
116 [ ${total} -eq 0 ] && break
117 lbase=$((lbase + usize))
122 format_start_mgs () {
123 do_rpc_nodes $(facet_active_host mgs) load_modules_local
125 start mgs $(mgsdevname) $MGS_MOUNT_OPTS
133 combined_mgs_mds || format_start_mgs
134 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
135 format_mdt $(facet_number $SINGLEMDS)
137 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
138 local nfiles=$((i - BCOUNT))
140 echo "+++ start to create for ${i} files set at: $(date) +++"
141 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
142 error "Fail to create files!"
143 echo "+++ end to create for ${i} files set at: $(date) +++"
146 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
147 error "Fail to start MDS!"
149 echo "start lfsck_namespace for ${i} files set at: $(date)"
150 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
153 local STATUS=$($SHOW_NAMESPACE |
154 awk '/^status/ { print $2 }')
155 [ "$STATUS" == "completed" ] && break
156 sleep 3 # check status every 3 seconds
159 echo "end lfsck_namespace for ${i} files set at: $(date)"
160 SPEED=$($SHOW_NAMESPACE |
161 awk '/^average_speed_phase1/ { print $2 }')
162 echo "lfsck_namespace speed is ${SPEED}/sec"
163 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
165 combined_mgs_mds || stop mgs
167 run_test 0 "lfsck namespace performance (routine case) without load"
170 [ "$mds1_FSTYPE" != ldiskfs ] &&
171 skip_env "not implemented for ZFS"
177 combined_mgs_mds || format_start_mgs
178 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
179 format_mdt $(facet_number $SINGLEMDS)
181 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
182 i = $((i * FACTOR)))); do
183 local nfiles=$((i - BCOUNT))
185 echo "+++ start to create for ${i} files set at: $(date) +++"
186 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
187 error "Fail to create files!"
188 echo "+++ end to create for ${i} files set at: $(date) +++"
191 local stime=$(date +%s)
192 echo "backup/restore ${i} files start at: $(date)"
193 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
194 echo "backup/restore ${i} files end at: $(date)"
195 local etime=$(date +%s)
196 local delta=$((etime - stime))
197 [ $delta -gt 0 ] || delta=1
198 echo "backup/restore ${i} files used ${delta} seconds"
199 echo "backup/restore speed is $((i / delta))/sec"
201 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
202 error "Fail to start MDS!"
204 echo "start lfsck_namespace for ${i} files set at: $(date)"
205 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
208 local STATUS=$($SHOW_NAMESPACE |
209 awk '/^status/ { print $2 }')
210 [ "$STATUS" == "completed" ] && break
211 sleep 3 # check status every 3 seconds
214 echo "end lfsck_namespace for ${i} files set at: $(date)"
215 local SPEED=$($SHOW_NAMESPACE |
216 awk '/^average_speed_phase1/ { print $2 }')
217 echo "lfsck_namespace speed is ${SPEED}/sec"
218 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
220 combined_mgs_mds || stop mgs
222 run_test 1 "lfsck namespace performance (backup/restore) without load"
227 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
228 i = $((i * FACTOR)))); do
230 combined_mgs_mds || format_start_mgs
231 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
232 format_mdt $(facet_number $SINGLEMDS)
234 echo "+++ start to create for ${i} files set at: $(date) +++"
235 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
236 error "Fail to create files!"
237 echo "+++ end to create for ${i} files set at: $(date) +++"
239 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
240 error "Fail to start MDS!"
242 echo "start lfsck_namespace for ${i} files set at: $(date)"
243 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
246 local STATUS=$($SHOW_NAMESPACE |
247 awk '/^status/ { print $2 }')
248 [ "$STATUS" == "completed" ] && break
249 sleep 3 # check status every 3 seconds
252 echo "end lfsck_namespace for ${i} files set at: $(date)"
253 local SPEED=$($SHOW_NAMESPACE |
254 awk '/^average_speed_phase1/ { print $2 }')
255 echo "lfsck_namespace speed is ${SPEED}/sec"
256 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
258 combined_mgs_mds || stop mgs
260 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
263 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
265 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
270 combined_mgs_mds || format_start_mgs
271 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
272 format_mdt $(facet_number $SINGLEMDS)
274 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
275 local nfiles=$((i - BCOUNT))
277 echo "+++ start to create for ${i} files set at: $(date) +++"
278 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
279 error "Fail to create files!"
280 echo "+++ end to create for ${i} files set at: $(date) +++"
284 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
285 error "Fail to start MDS!"
287 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
288 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
291 local STATUS=$($SHOW_NAMESPACE |
292 awk '/^status/ { print $2 }')
293 [ "$STATUS" == "completed" ] && break
294 sleep 3 # check status every 3 seconds
297 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
298 local FULL_SPEED=$($SHOW_NAMESPACE |
299 awk '/^average_speed_phase1/ { print $2 }')
300 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
301 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
302 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
305 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
306 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
307 error "Fail to start MDS!"
309 $STOP_LFSCK > /dev/null 2>&1
310 echo "start lfsck_namespace with speed ${j} at: $(date)"
311 $START_NAMESPACE --reset -s ${j} ||
312 error "Fail to start lfsck_namespace with speed ${j}!"
313 # lfsck_namespace will be paused when MDS stop,
314 # and will be restarted automatically when mount up again.
315 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
317 local nfiles=$(((i - BCOUNT) / 2))
319 echo "+++ start to create for ${i} files set at: $(date) +++"
320 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
321 error "Fail to create files!"
322 echo "+++ end to create for ${i} files set at: $(date) +++"
327 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
328 error "Fail to start MDS!"
330 $STOP_LFSCK /dev/null 2>&1
331 echo "start lfsck_namespace with full speed at: $(date)"
332 $START_NAMESPACE --reset -s 0 ||
333 error "Fail to start lfsck_namespace with full speed!"
334 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
336 local nfiles=$(((i - BCOUNT) / 2))
338 echo "+++ start to create for ${i} files set at: $(date) +++"
339 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
340 error "Fail to create files!"
341 echo "+++ end to create for ${i} files set at: $(date) +++"
342 combined_mgs_mds || stop mgs
344 run_test 3 "lfsck namespace impact on create performance"
350 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
354 echo "***** Start layout LFSCK on all devices at: $(date) *****"
355 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
357 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
358 mdd.${MDT_DEV}.lfsck_layout |
359 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
363 echo "***** End layout LFSCK on all devices at: $(date) *****"
365 for n in $(seq $MDSCOUNT); do
368 local SPEED=$(show_layout ${n} |
369 awk '/^average_speed_phase1/ { print $2 }')
371 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
379 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
380 local basedir="$mntpt/$tdir/$idx1/$idx2"
383 error_noexit "(11) Fail to mkdir $mntpt"
387 mount_client $mntpt || {
388 error_noexit "(12) Fail to mount $mntpt"
394 error_noexit "(13) Fail to mkdir $basedir"
398 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
399 createmany -o ${basedir}/f $UNIT || {
401 error_noexit "(14) Fail to gen $UNIT files under $basedir"
404 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
412 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
413 for ((k = 0; k < $MDSCOUNT; k++)); do
414 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
416 for ((l = 1; l <= $cnt; l++)); do
417 layout_gen_one ${k} ${l} &
422 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
426 local saved_mdscount=$MDSCOUNT
427 local saved_ostcount=$OSTCOUNT
430 stopall > /dev/null || error "(1) Fail to stopall"
432 LFSCKDIR="$DIR/$tdir"
434 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
437 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
440 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
442 formatall > /dev/null ||
443 error "(2) Fail to formatall, subdirs=${j}"
446 setupall > /dev/null ||
447 error "(3) Fail to setupall, subdirs=${j}"
450 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
452 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
453 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
456 layout_gen_set ${j} || RC=$?
458 error "(6) generate set $RC, subdirs=${j}"
461 layout_test_one || RC=$?
463 error "(7) LFSCK failed with $RC, subdirs=${j}"
467 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
470 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
473 MDSCOUNT=$saved_mdscount
474 OSTCOUNT=$saved_ostcount
480 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
483 echo "Inject failure stub to simulate dangling reference"
484 #define OBD_FAIL_LFSCK_DANGLING 0x1610
485 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
489 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
492 local saved_mdscount=$MDSCOUNT
495 stopall > /dev/null || error "(1) Fail to stopall"
497 LFSCKDIR="$DIR/$tdir"
498 for ((i = 1; i <= $saved_mdscount; i++)); do
501 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
504 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
506 formatall > /dev/null ||
507 error "(2) Fail to formatall, subdirs=${j}"
510 setupall > /dev/null ||
511 error "(3) Fail to setupall, subdirs=${j}"
514 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
516 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
517 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
520 layout_gen_set ${j} || RC=$?
522 error "(6) generate set $RC, subdirs=${j}"
525 layout_test_one || RC=$?
527 error "(7) LFSCK failed with $RC, subdirs=${j}"
531 stopall > /dev/null || error "(8) Fail to stopall"
534 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
537 MDSCOUNT=$saved_mdscount
543 run_test 5a "lfsck layout performance (routine case) without load for DNE"
546 echo "Inject failure stub to simulate dangling reference"
547 #define OBD_FAIL_LFSCK_DANGLING 0x1610
548 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
552 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
554 lfsck_fast_create() {
558 local ldir="/test-${lbase}"
563 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
564 local usize=$((count / NTHREADS))
565 [ ${usize} -eq 0 ] && break
566 local tdir=${ldir}-${cycle}-
568 echo "[cycle: $cycle] [threads: $threads]"\
569 "[files: $count] [basedir: $tdir]"
573 total=$((total - usize * NTHREADS))
574 [ $total -eq 0 ] && break
575 lbase=$((lbase + usize))
580 lfsck_detach_error() {
586 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
589 stopall > /dev/null || error "(1) Fail to stopall"
591 local saved_mdscount=$MDSCOUNT
593 LFSCKDIR="$DIR/$tdir"
596 formatall > /dev/null || error "(2) Fail to formatall"
599 setupall > /dev/null || error "(3) Fail to setupall"
601 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
603 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
604 error "(5) Fail to setstripe on $LFSCKDIR"
607 layout_gen_set $TOTSUBDIR || RC=$?
609 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
612 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
613 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
614 error "(7) Fail to start layout LFSCK"
616 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
617 mdd.${MDT_DEV}.lfsck_layout |
618 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
620 error "(8) layout LFSCK cannot finished in time"
622 echo "***** End layout LFSCK on single MDS at: $(date) *****"
624 local SPEED=$(show_layout 1 |
625 awk '/^average_speed_phase1/ { print $2 }')
626 echo "lfsck_layout full_speed is $SPEED objs/sec"
628 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
629 local nfiles=$((inc_count / 2))
632 for ((m = 0, n = $INCFACTOR; n < 100;
633 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
634 local sl=$((SPEED * n / 100))
636 $STOP_LFSCK > /dev/null 2>&1
638 echo "start lfsck_layout with speed ${sl} at: $(date)"
639 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
641 "(9) Fail to start lfsck_layout with speed ${sl}"
643 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
644 lfsck_fast_create $nfiles ${m} $NTHREADS ||
645 lfsck_detach_error "(10) Fail to create files"
646 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
649 $STOP_LFSCK > /dev/null 2>&1
651 echo "start lfsck_layout with full speed at: $(date)"
652 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
654 "(11) Fail to start lfsck_layout with full speed"
656 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
657 lfsck_fast_create $nfiles ${m} $NTHREADS ||
658 lfsck_detach_error "(12) Fail to create files"
659 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
662 $STOP_LFSCK > /dev/null 2>&1
664 echo "create without lfsck_layout run back-ground"
665 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
666 lfsck_fast_create $nfiles ${m} $NTHREADS ||
667 lfsck_detach_error "(13) Fail to create files"
668 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
673 stopall > /dev/null || error "(14) Fail to stopall"
675 MDSCOUNT=$saved_mdscount
677 run_test 6 "lfsck layout impact on create performance"
683 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
686 namespace_test_one() {
687 echo "***** Start namespace LFSCK on all devices at: $(date) *****"
688 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
690 for n in $(seq $MDSCOUNT); do
691 wait_update_facet mds${n} "$LCTL get_param -n \
692 mdd.$(facet_svc mds${n}).lfsck_namespace |
693 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
698 echo "***** End namespace LFSCK on all devices at: $(date) *****"
700 for n in $(seq $MDSCOUNT); do
703 local SPEED=$(show_namespace ${n} |
704 awk '/^average_speed_total/ { print $2 }')
706 echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
711 namespace_gen_one() {
714 local idx3=$(((idx1 + 1) % MDSCOUNT))
715 local base_mntpt="/mnt/lustre_lfsck_${idx1}"
716 local show_dir="$LFSCKDIR/${idx1}/${idx2}"
717 local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
719 mkdir $show_dir || return 20
721 local count=$((UNIT * 78 / 100)) # 78% regular files
722 local sub_count=$((count / NTHREADS))
723 echo "Creating $count regular files under $show_dir at: $(date)"
724 for ((m = 0; m < $NTHREADS; m++)); do
725 local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
727 createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
731 error_noexit "(21) Fail to gen regular files under $show_dir"
735 count=$((UNIT * 3 / 100)) # 3% local sub-dirs
736 echo "Creating $count local sub-dirs under $show_dir at: $(date)"
737 createmany -d $work_dir/d_l_ $count > /dev/null || {
738 error_noexit "(22) Fail to gen local sub-dir under $show_dir"
742 # 3% * 5 = 15% regular files under local sub-dirs
743 echo "Creating 5 regular files under each local sub-dir at: $(date)"
744 for ((m = 0; m < $count; m++)); do
745 createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
747 "(23) Fail to gen regular under $work_dir/d_l_${m}"
752 count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
753 echo "Creating $count multiple hard-links under $show_dir at: $(date)"
754 for ((m = 0; m < $count; m++)); do
755 ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
757 "(24) Fail to hardlink to $work_dir/f_0_${m}"
762 count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
763 echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
764 "files under each remote sub-dir at: $(date)"
765 for ((m = 0; m < $count; m++)); do
766 $LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
768 "(25) Fail to remote mkdir $work_dir/d_r_${m}"
772 # 0.3% * 4 = 1.2% regular files under remote sub-dirs
773 createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
775 "(26) Fail to gen regular under $work_dir/d_r_${m}"
780 # 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
781 count=$((UNIT * 3 / 1000))
782 echo "Creating $count 2-striped sub-dirs under $show_dir," \
783 "and 4 regular files under each striped sub-dir at: $(date)"
784 for ((m = 0; m < $count; m++)); do
785 $LFS setdirstripe -i ${idx1} -c 2 -H all_char \
786 $work_dir/d_s_${m} || {
788 "(27) Fail to make striped-dir $work_dir/d_s_${m}"
792 # 0.3% * 4 = 1.2% regular files under striped sub-dirs
793 createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
795 "(28) Fail to gen regular under $work_dir/d_s_${m}"
801 namespace_gen_mdt() {
804 local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
806 $LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
808 for ((m = 0; m < $NTHREADS; m++)); do
809 local mntpt="${base_mntpt}_${m}"
812 umount ${base_mntpt}_*
813 error_noexit "(11) Fail to mkdir $mntpt"
817 mount_client $mntpt || {
818 umount ${base_mntpt}_*
819 error_noexit "(12) Fail to mount $mntpt"
824 for ((l = 0; l < $dir_cnt; l++)); do
825 namespace_gen_one ${mdt_idx} ${l}
828 umount ${base_mntpt}_*
831 namespace_gen_set() {
835 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
836 for ((k = 0; k < $MDSCOUNT; k++)); do
837 namespace_gen_mdt ${k} ${cnt} &
840 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
845 local saved_mdscount=$MDSCOUNT
847 [ $MDSCOUNT -le 8 ] ||
848 error "Too much MDT, test data set on each MDT may be unbalance"
851 stopall > /dev/null || error "(1) Fail to stopall"
853 LFSCKDIR="$DIR/$tdir"
854 for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
857 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
860 for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
861 j = $((j + MINSUBDIR)))); do
863 formatall > /dev/null ||
864 error "(2) Fail to formatall, subdirs=${j}"
867 setupall > /dev/null ||
868 error "(3) Fail to setupall, subdirs=${j}"
871 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
873 $LFS setstripe -c 1 -i -1 $LFSCKDIR ||
874 error "(5) Fail to setstripe on $LFSCKDIR"
876 do_nodes $(comma_list $(mdts_nodes)) \
877 $LCTL set_param fail_loc=$local_loc
880 namespace_gen_set ${j} || RC=$?
882 error "(6) generate set $RC, subdirs=${j}"
885 namespace_test_one || RC=$?
887 error "(7) LFSCK failed with $RC, subdirs=${j}"
889 do_nodes $(comma_list $(mdts_nodes)) \
890 $LCTL set_param fail_loc=0
894 stopall > /dev/null || error "(8) Fail to stopall"
897 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
900 MDSCOUNT=$saved_mdscount
906 run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
909 echo "Inject failure stub to simulate the case of lost linkEA"
910 #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
913 run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
916 echo "Inject failure stub to simulate the case of bad FID-in-dirent"
917 #define OBD_FAIL_FID_INDIR 0x1501
920 run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
923 [ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
925 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
928 stopall > /dev/null || error "(1) Fail to stopall"
930 local saved_mdscount=$MDSCOUNT
932 LFSCKDIR="$DIR/$tdir"
935 formatall > /dev/null || error "(2) Fail to formatall"
938 setupall > /dev/null || error "(3) Fail to setupall"
940 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
942 $LFS setstripe -c 1 -i 0 $LFSCKDIR ||
943 error "(5) Fail to setstripe on $LFSCKDIR"
946 namespace_gen_set $TOTSUBDIR || RC=$?
948 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
951 echo "***** Start namespace LFSCK at: $(date) *****"
952 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
953 error "(7) Fail to start namespace LFSCK"
955 for n in $(seq $MDSCOUNT); do
956 wait_update_facet mds${n} "$LCTL get_param -n \
957 mdd.$(facet_svc mds${n}).lfsck_namespace |
958 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
960 error "(8) namespace LFSCK cannot finished in time"
963 echo "***** End namespace LFSCK at: $(date) *****"
965 local SPEED=$(show_namespace 1 |
966 awk '/^average_speed_phase1/ { print $2 }')
967 echo "lfsck_namespace full_speed is $SPEED objs/sec"
970 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
971 local nfiles=$((inc_count / 2))
976 local stime=$(date +%s)
977 lfsck_fast_create $nfiles ${m} $NTHREADS ||
978 lfsck_detach_error "(9) Fail to create files"
979 local etime=$(date +%s)
980 echo "created $nfiles without lfsck_namespace run back-ground used" \
981 "$((etime - stime)) seconds"
984 for ((m = nfiles, n = $INCFACTOR; n < 100;
985 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
986 local sl=$((SPEED * n / 100))
988 $STOP_LFSCK > /dev/null 2>&1
989 echo "start lfsck_namespace with speed ${sl} at: $(date)"
990 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
992 "(10) Fail to start lfsck_namespace with speed ${sl}"
995 lfsck_fast_create $nfiles ${m} $NTHREADS ||
996 lfsck_detach_error "(11) Fail to create files"
998 echo "created $nfiles with namespace LFSCK run with the" \
999 "speed limit of ${n}% of full speed used" \
1000 "$((etime - stime)) seconds"
1004 $STOP_LFSCK > /dev/null 2>&1
1005 echo "start lfsck_namespace with full speed at: $(date)"
1006 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
1007 lfsck_detach_error \
1008 "(12) Fail to start lfsck_namespace with full speed"
1011 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1012 lfsck_detach_error "(13) Fail to create files"
1014 echo "created $nfiles with namespace LFSCK run with full speed used" \
1015 "$((etime - stime)) seconds"
1018 $STOP_LFSCK > /dev/null 2>&1
1023 stopall > /dev/null || error "(14) Fail to stopall"
1025 MDSCOUNT=$saved_mdscount
1027 run_test 8 "lfsck namespace impact on create performance"
1029 # cleanup the system at last
1032 check_and_cleanup_lustre