6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 require_dsh_mds || exit 0
17 require_dsh_ost || exit 0
20 skip "skip lfsck performance test under non-SLOW mode" && exit 0
22 NTHREADS=${NTHREADS:-0}
24 MINCOUNT=${MINCOUNT:-4096}
25 MAXCOUNT=${MAXCOUNT:-8192}
26 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
27 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
28 BASE_COUNT=${BASE_COUNT:-8192}
30 INCFACTOR=${INCFACTOR:-25} #percent
31 MINSUBDIR=${MINSUBDIR:-1}
32 MAXSUBDIR=${MAXSUBDIR:-2}
33 TOTSUBDIR=${TOTSUBDIR:-2}
36 RCMD="do_facet ${SINGLEMDS}"
37 RLCTL="${RCMD} ${LCTL}"
38 MDT_DEV="${FSNAME}-MDT0000"
39 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
40 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
42 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
44 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
46 if [ ${NTHREADS} -eq 0 ]; then
47 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
48 NTHREADS=$((CPUCORE * 2))
52 ${RCMD} "modprobe obdecho"
54 ${ECHOCMD} "${LCTL} <<-EOF
55 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
61 ${ECHOCMD} "${LCTL} <<-EOF
69 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
72 ${ECHOCMD} "${LCTL} <<-EOF
77 for ((j = 1; j < ${threads}; j++)); do
78 ${ECHOCMD} "${LCTL} <<-EOF
80 test_mkdir ${tdir}${j}
84 ${ECHOCMD} "${LCTL} <<-EOF
86 --threads ${threads} 0 ${echodev} test_create \
87 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
92 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
96 lfsck_create_nfiles() {
101 local ldir="/test-${lbase}"
106 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
107 local usize=$((count / NTHREADS))
108 [ ${usize} -eq 0 ] && break
109 local tdir=${ldir}-${cycle}-
111 echo "[cycle: ${cycle}] [threads: ${threads}]"\
112 "[files: ${count}] [basedir: ${tdir}]"
113 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
114 error "Fail to start MDS!"
115 #define OBD_FAIL_FID_IGIF 0x1504
116 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
122 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
123 stop ${SINGLEMDS} || error "Fail to stop MDS!"
125 total=$((total - usize * NTHREADS))
126 [ ${total} -eq 0 ] && break
127 lbase=$((lbase + usize))
139 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
140 format_mdt $(facet_number $SINGLEMDS)
142 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
143 local nfiles=$((i - BCOUNT))
145 echo "+++ start to create for ${i} files set at: $(date) +++"
146 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
147 error "Fail to create files!"
148 echo "+++ end to create for ${i} files set at: $(date) +++"
151 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
152 error "Fail to start MDS!"
154 echo "start lfsck_namespace for ${i} files set at: $(date)"
155 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
158 local STATUS=$($SHOW_NAMESPACE |
159 awk '/^status/ { print $2 }')
160 [ "$STATUS" == "completed" ] && break
161 sleep 3 # check status every 3 seconds
164 echo "end lfsck_namespace for ${i} files set at: $(date)"
165 SPEED=$($SHOW_NAMESPACE |
166 awk '/^average_speed_phase1/ { print $2 }')
167 echo "lfsck_namespace speed is ${SPEED}/sec"
168 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
171 run_test 0 "lfsck namespace performance (routine case) without load"
174 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
175 skip "not implemented for ZFS" && return
181 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
182 format_mdt $(facet_number $SINGLEMDS)
184 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
185 i = $((i * FACTOR)))); do
186 local nfiles=$((i - BCOUNT))
188 echo "+++ start to create for ${i} files set at: $(date) +++"
189 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
190 error "Fail to create files!"
191 echo "+++ end to create for ${i} files set at: $(date) +++"
194 local stime=$(date +%s)
195 echo "backup/restore ${i} files start at: $(date)"
196 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
197 echo "backup/restore ${i} files end at: $(date)"
198 local etime=$(date +%s)
199 local delta=$((etime - stime))
200 [ $delta -gt 0 ] || delta=1
201 echo "backup/restore ${i} files used ${delta} seconds"
202 echo "backup/restore speed is $((i / delta))/sec"
204 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
205 error "Fail to start MDS!"
207 echo "start lfsck_namespace for ${i} files set at: $(date)"
208 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
211 local STATUS=$($SHOW_NAMESPACE |
212 awk '/^status/ { print $2 }')
213 [ "$STATUS" == "completed" ] && break
214 sleep 3 # check status every 3 seconds
217 echo "end lfsck_namespace for ${i} files set at: $(date)"
218 local SPEED=$($SHOW_NAMESPACE |
219 awk '/^average_speed_phase1/ { print $2 }')
220 echo "lfsck_namespace speed is ${SPEED}/sec"
221 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
224 run_test 1 "lfsck namespace performance (backup/restore) without load"
229 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
230 i = $((i * FACTOR)))); do
232 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
233 format_mdt $(facet_number $SINGLEMDS)
235 echo "+++ start to create for ${i} files set at: $(date) +++"
236 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
237 error "Fail to create files!"
238 echo "+++ end to create for ${i} files set at: $(date) +++"
240 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
241 error "Fail to start MDS!"
243 echo "start lfsck_namespace for ${i} files set at: $(date)"
244 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
247 local STATUS=$($SHOW_NAMESPACE |
248 awk '/^status/ { print $2 }')
249 [ "$STATUS" == "completed" ] && break
250 sleep 3 # check status every 3 seconds
253 echo "end lfsck_namespace for ${i} files set at: $(date)"
254 local SPEED=$($SHOW_NAMESPACE |
255 awk '/^average_speed_phase1/ { print $2 }')
256 echo "lfsck_namespace speed is ${SPEED}/sec"
257 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
260 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
263 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
265 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
270 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
271 format_mdt $(facet_number $SINGLEMDS)
273 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
274 local nfiles=$((i - BCOUNT))
276 echo "+++ start to create for ${i} files set at: $(date) +++"
277 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
278 error "Fail to create files!"
279 echo "+++ end to create for ${i} files set at: $(date) +++"
283 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
284 error "Fail to start MDS!"
286 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
287 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
290 local STATUS=$($SHOW_NAMESPACE |
291 awk '/^status/ { print $2 }')
292 [ "$STATUS" == "completed" ] && break
293 sleep 3 # check status every 3 seconds
296 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
297 local FULL_SPEED=$($SHOW_NAMESPACE |
298 awk '/^average_speed_phase1/ { print $2 }')
299 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
300 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
301 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
304 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
305 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
306 error "Fail to start MDS!"
308 $STOP_LFSCK > /dev/null 2>&1
309 echo "start lfsck_namespace with speed ${j} at: $(date)"
310 $START_NAMESPACE --reset -s ${j} ||
311 error "Fail to start lfsck_namespace with speed ${j}!"
312 # lfsck_namespace will be paused when MDS stop,
313 # and will be restarted automatically when mount up again.
314 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
316 local nfiles=$(((i - BCOUNT) / 2))
318 echo "+++ start to create for ${i} files set at: $(date) +++"
319 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
320 error "Fail to create files!"
321 echo "+++ end to create for ${i} files set at: $(date) +++"
326 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
327 error "Fail to start MDS!"
329 $STOP_LFSCK /dev/null 2>&1
330 echo "start lfsck_namespace with full speed at: $(date)"
331 $START_NAMESPACE --reset -s 0 ||
332 error "Fail to start lfsck_namespace with full speed!"
333 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
335 local nfiles=$(((i - BCOUNT) / 2))
337 echo "+++ start to create for ${i} files set at: $(date) +++"
338 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
339 error "Fail to create files!"
340 echo "+++ end to create for ${i} files set at: $(date) +++"
342 run_test 3 "lfsck namespace impact on create performance"
348 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
352 echo "***** Start layout LFSCK on all devices at: $(date) *****"
353 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
355 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
356 mdd.${MDT_DEV}.lfsck_layout |
357 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
361 echo "***** End layout LFSCK on all devices at: $(date) *****"
363 for n in $(seq $MDSCOUNT); do
366 local SPEED=$(show_layout ${n} |
367 awk '/^average_speed_phase1/ { print $2 }')
369 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
377 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
378 local basedir="$mntpt/$tdir/$idx1/$idx2"
381 error_noexit "(11) Fail to mkdir $mntpt"
385 mount_client $mntpt || {
386 error_noexit "(12) Fail to mount $mntpt"
392 error_noexit "(13) Fail to mkdir $basedir"
396 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
397 createmany -o ${basedir}/f $UNIT || {
399 error_noexit "(14) Fail to gen $UNIT files under $basedir"
402 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
410 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
411 for ((k = 0; k < $MDSCOUNT; k++)); do
412 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
414 for ((l = 1; l <= $cnt; l++)); do
415 layout_gen_one ${k} ${l} &
420 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
424 local saved_mdscount=$MDSCOUNT
425 local saved_ostcount=$OSTCOUNT
428 stopall > /dev/null || error "(1) Fail to stopall"
430 LFSCKDIR="$DIR/$tdir"
432 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
435 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
438 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
440 formatall > /dev/null ||
441 error "(2) Fail to formatall, subdirs=${j}"
444 setupall > /dev/null ||
445 error "(3) Fail to setupall, subdirs=${j}"
448 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
450 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
451 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
454 layout_gen_set ${j} || RC=$?
456 error "(6) generate set $RC, subdirs=${j}"
459 layout_test_one || RC=$?
461 error "(7) LFSCK failed with $RC, subdirs=${j}"
465 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
468 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
471 MDSCOUNT=$saved_mdscount
472 OSTCOUNT=$saved_ostcount
478 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
481 echo "Inject failure stub to simulate dangling reference"
482 #define OBD_FAIL_LFSCK_DANGLING 0x1610
483 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
487 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
490 local saved_mdscount=$MDSCOUNT
493 stopall > /dev/null || error "(1) Fail to stopall"
495 LFSCKDIR="$DIR/$tdir"
496 for ((i = 1; i <= $saved_mdscount; i++)); do
499 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
502 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
504 formatall > /dev/null ||
505 error "(2) Fail to formatall, subdirs=${j}"
508 setupall > /dev/null ||
509 error "(3) Fail to setupall, subdirs=${j}"
512 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
514 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
515 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
518 layout_gen_set ${j} || RC=$?
520 error "(6) generate set $RC, subdirs=${j}"
523 layout_test_one || RC=$?
525 error "(7) LFSCK failed with $RC, subdirs=${j}"
529 stopall > /dev/null || error "(8) Fail to stopall"
532 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
535 MDSCOUNT=$saved_mdscount
541 run_test 5a "lfsck layout performance (routine case) without load for DNE"
544 echo "Inject failure stub to simulate dangling reference"
545 #define OBD_FAIL_LFSCK_DANGLING 0x1610
546 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
550 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
552 lfsck_fast_create() {
556 local ldir="/test-${lbase}"
561 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
562 local usize=$((count / NTHREADS))
563 [ ${usize} -eq 0 ] && break
564 local tdir=${ldir}-${cycle}-
566 echo "[cycle: $cycle] [threads: $threads]"\
567 "[files: $count] [basedir: $tdir]"
571 total=$((total - usize * NTHREADS))
572 [ $total -eq 0 ] && break
573 lbase=$((lbase + usize))
578 lfsck_detach_error() {
584 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
587 stopall > /dev/null || error "(1) Fail to stopall"
589 local saved_mdscount=$MDSCOUNT
591 LFSCKDIR="$DIR/$tdir"
594 formatall > /dev/null || error "(2) Fail to formatall"
597 setupall > /dev/null || error "(3) Fail to setupall"
599 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
601 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
602 error "(5) Fail to setstripe on $LFSCKDIR"
605 layout_gen_set $TOTSUBDIR || RC=$?
607 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
610 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
611 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
612 error "(7) Fail to start layout LFSCK"
614 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
615 mdd.${MDT_DEV}.lfsck_layout |
616 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
618 error "(8) layout LFSCK cannot finished in time"
620 echo "***** End layout LFSCK on single MDS at: $(date) *****"
622 local SPEED=$(show_layout 1 |
623 awk '/^average_speed_phase1/ { print $2 }')
624 echo "lfsck_layout full_speed is $SPEED objs/sec"
626 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
627 local nfiles=$((inc_count / 2))
630 for ((m = 0, n = $INCFACTOR; n < 100;
631 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
632 local sl=$((SPEED * n / 100))
634 $STOP_LFSCK > /dev/null 2>&1
636 echo "start lfsck_layout with speed ${sl} at: $(date)"
637 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
639 "(9) Fail to start lfsck_layout with speed ${sl}"
641 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
642 lfsck_fast_create $nfiles ${m} $NTHREADS ||
643 lfsck_detach_error "(10) Fail to create files"
644 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
647 $STOP_LFSCK > /dev/null 2>&1
649 echo "start lfsck_layout with full speed at: $(date)"
650 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
652 "(11) Fail to start lfsck_layout with full speed"
654 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
655 lfsck_fast_create $nfiles ${m} $NTHREADS ||
656 lfsck_detach_error "(12) Fail to create files"
657 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
660 $STOP_LFSCK > /dev/null 2>&1
662 echo "create without lfsck_layout run back-ground"
663 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
664 lfsck_fast_create $nfiles ${m} $NTHREADS ||
665 lfsck_detach_error "(13) Fail to create files"
666 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
671 stopall > /dev/null || error "(14) Fail to stopall"
673 MDSCOUNT=$saved_mdscount
675 run_test 6 "lfsck layout impact on create performance"
681 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
684 namespace_test_one() {
685 echo "***** Start namespace LFSCK on all devices at: $(date) *****"
686 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
688 for n in $(seq $MDSCOUNT); do
689 wait_update_facet mds${n} "$LCTL get_param -n \
690 mdd.$(facet_svc mds${n}).lfsck_namespace |
691 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
696 echo "***** End namespace LFSCK on all devices at: $(date) *****"
698 for n in $(seq $MDSCOUNT); do
701 local SPEED=$(show_namespace ${n} |
702 awk '/^average_speed_total/ { print $2 }')
704 echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
709 namespace_gen_one() {
712 local idx3=$(((idx1 + 1) % MDSCOUNT))
713 local base_mntpt="/mnt/lustre_lfsck_${idx1}"
714 local show_dir="$LFSCKDIR/${idx1}/${idx2}"
715 local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
717 mkdir $show_dir || return 20
719 local count=$((UNIT * 78 / 100)) # 78% regular files
720 local sub_count=$((count / NTHREADS))
721 echo "Creating $count regular files under $show_dir at: $(date)"
722 for ((m = 0; m < $NTHREADS; m++)); do
723 local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
725 createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
729 error_noexit "(21) Fail to gen regular files under $show_dir"
733 count=$((UNIT * 3 / 100)) # 3% local sub-dirs
734 echo "Creating $count local sub-dirs under $show_dir at: $(date)"
735 createmany -d $work_dir/d_l_ $count > /dev/null || {
736 error_noexit "(22) Fail to gen local sub-dir under $show_dir"
740 # 3% * 5 = 15% regular files under local sub-dirs
741 echo "Creating 5 regular files under each local sub-dir at: $(date)"
742 for ((m = 0; m < $count; m++)); do
743 createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
745 "(23) Fail to gen regular under $work_dir/d_l_${m}"
750 count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
751 echo "Creating $count multiple hard-links under $show_dir at: $(date)"
752 for ((m = 0; m < $count; m++)); do
753 ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
755 "(24) Fail to hardlink to $work_dir/f_0_${m}"
760 count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
761 echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
762 "files under each remote sub-dir at: $(date)"
763 for ((m = 0; m < $count; m++)); do
764 $LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
766 "(25) Fail to remote mkdir $work_dir/d_r_${m}"
770 # 0.3% * 4 = 1.2% regular files under remote sub-dirs
771 createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
773 "(26) Fail to gen regular under $work_dir/d_r_${m}"
778 # 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
779 count=$((UNIT * 3 / 1000))
780 echo "Creating $count 2-striped sub-dirs under $show_dir," \
781 "and 4 regular files under each striped sub-dir at: $(date)"
782 for ((m = 0; m < $count; m++)); do
783 $LFS setdirstripe -i ${idx1} -c 2 -t all_char \
784 $work_dir/d_s_${m} || {
786 "(27) Fail to make striped-dir $work_dir/d_s_${m}"
790 # 0.3% * 4 = 1.2% regular files under striped sub-dirs
791 createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
793 "(28) Fail to gen regular under $work_dir/d_s_${m}"
799 namespace_gen_mdt() {
802 local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
804 $LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
806 for ((m = 0; m < $NTHREADS; m++)); do
807 local mntpt="${base_mntpt}_${m}"
810 umount ${base_mntpt}_*
811 error_noexit "(11) Fail to mkdir $mntpt"
815 mount_client $mntpt || {
816 umount ${base_mntpt}_*
817 error_noexit "(12) Fail to mount $mntpt"
822 for ((l = 0; l < $dir_cnt; l++)); do
823 namespace_gen_one ${mdt_idx} ${l}
826 umount ${base_mntpt}_*
829 namespace_gen_set() {
833 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
834 for ((k = 0; k < $MDSCOUNT; k++)); do
835 namespace_gen_mdt ${k} ${cnt} &
838 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
843 local saved_mdscount=$MDSCOUNT
845 [ $MDSCOUNT -le 8 ] ||
846 error "Too much MDT, test data set on each MDT may be unbalance"
849 stopall > /dev/null || error "(1) Fail to stopall"
851 LFSCKDIR="$DIR/$tdir"
852 for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
855 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
858 for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
859 j = $((j + MINSUBDIR)))); do
861 formatall > /dev/null ||
862 error "(2) Fail to formatall, subdirs=${j}"
865 setupall > /dev/null ||
866 error "(3) Fail to setupall, subdirs=${j}"
869 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
871 $LFS setstripe -c 1 -i -1 $LFSCKDIR ||
872 error "(5) Fail to setstripe on $LFSCKDIR"
874 do_nodes $(comma_list $(mdts_nodes)) \
875 $LCTL set_param fail_loc=$local_loc
878 namespace_gen_set ${j} || RC=$?
880 error "(6) generate set $RC, subdirs=${j}"
883 namespace_test_one || RC=$?
885 error "(7) LFSCK failed with $RC, subdirs=${j}"
887 do_nodes $(comma_list $(mdts_nodes)) \
888 $LCTL set_param fail_loc=0
892 stopall > /dev/null || error "(8) Fail to stopall"
895 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
898 MDSCOUNT=$saved_mdscount
904 run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
907 echo "Inject failure stub to simulate the case of lost linkEA"
908 #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
911 run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
914 echo "Inject failure stub to simulate the case of bad FID-in-dirent"
915 #define OBD_FAIL_FID_INDIR 0x1501
918 run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
921 [ $MDSCOUNT -lt 2 ] &&
922 skip "We need at least 2 MDSes for this test" && return
924 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
927 stopall > /dev/null || error "(1) Fail to stopall"
929 local saved_mdscount=$MDSCOUNT
931 LFSCKDIR="$DIR/$tdir"
934 formatall > /dev/null || error "(2) Fail to formatall"
937 setupall > /dev/null || error "(3) Fail to setupall"
939 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
941 $LFS setstripe -c 1 -i 0 $LFSCKDIR ||
942 error "(5) Fail to setstripe on $LFSCKDIR"
945 namespace_gen_set $TOTSUBDIR || RC=$?
947 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
950 echo "***** Start namespace LFSCK at: $(date) *****"
951 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
952 error "(7) Fail to start namespace LFSCK"
954 for n in $(seq $MDSCOUNT); do
955 wait_update_facet mds${n} "$LCTL get_param -n \
956 mdd.$(facet_svc mds${n}).lfsck_namespace |
957 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
959 error "(8) namespace LFSCK cannot finished in time"
962 echo "***** End namespace LFSCK at: $(date) *****"
964 local SPEED=$(show_namespace 1 |
965 awk '/^average_speed_phase1/ { print $2 }')
966 echo "lfsck_namespace full_speed is $SPEED objs/sec"
969 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
970 local nfiles=$((inc_count / 2))
975 local stime=$(date +%s)
976 lfsck_fast_create $nfiles ${m} $NTHREADS ||
977 lfsck_detach_error "(9) Fail to create files"
978 local etime=$(date +%s)
979 echo "created $nfiles without lfsck_namespace run back-ground used" \
980 "$((etime - stime)) seconds"
983 for ((m = nfiles, n = $INCFACTOR; n < 100;
984 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
985 local sl=$((SPEED * n / 100))
987 $STOP_LFSCK > /dev/null 2>&1
988 echo "start lfsck_namespace with speed ${sl} at: $(date)"
989 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
991 "(10) Fail to start lfsck_namespace with speed ${sl}"
994 lfsck_fast_create $nfiles ${m} $NTHREADS ||
995 lfsck_detach_error "(11) Fail to create files"
997 echo "created $nfiles with namespace LFSCK run with the" \
998 "speed limit of ${n}% of full speed used" \
999 "$((etime - stime)) seconds"
1003 $STOP_LFSCK > /dev/null 2>&1
1004 echo "start lfsck_namespace with full speed at: $(date)"
1005 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
1006 lfsck_detach_error \
1007 "(12) Fail to start lfsck_namespace with full speed"
1010 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1011 lfsck_detach_error "(13) Fail to create files"
1013 echo "created $nfiles with namespace LFSCK run with full speed used" \
1014 "$((etime - stime)) seconds"
1017 $STOP_LFSCK > /dev/null 2>&1
1022 stopall > /dev/null || error "(14) Fail to stopall"
1024 MDSCOUNT=$saved_mdscount
1026 run_test 8 "lfsck namespace impact on create performance"
1028 # cleanup the system at last