7 LUSTRE=${LUSTRE:-$(dirname $0)/..}
8 . $LUSTRE/tests/test-framework.sh
12 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
15 require_dsh_mds || exit 0
16 require_dsh_ost || exit 0
19 skip "skip lfsck performance test under non-SLOW mode"
21 NTHREADS=${NTHREADS:-0}
23 MINCOUNT=${MINCOUNT:-4096}
24 MAXCOUNT=${MAXCOUNT:-8192}
25 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
26 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
27 BASE_COUNT=${BASE_COUNT:-8192}
29 INCFACTOR=${INCFACTOR:-25} #percent
30 MINSUBDIR=${MINSUBDIR:-1}
31 MAXSUBDIR=${MAXSUBDIR:-2}
32 TOTSUBDIR=${TOTSUBDIR:-2}
35 RCMD="do_facet ${SINGLEMDS}"
36 RLCTL="${RCMD} ${LCTL}"
37 MDT_DEV="${FSNAME}-MDT0000"
38 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
39 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
40 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
41 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
42 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
43 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
45 if [ ${NTHREADS} -eq 0 ]; then
46 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
47 NTHREADS=$((CPUCORE * 2))
51 ${RCMD} "modprobe obdecho"
53 ${ECHOCMD} "${LCTL} <<-EOF
54 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
60 ${ECHOCMD} "${LCTL} <<-EOF
68 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
71 ${ECHOCMD} "${LCTL} <<-EOF
76 for ((j = 1; j < ${threads}; j++)); do
77 ${ECHOCMD} "${LCTL} <<-EOF
79 test_mkdir ${tdir}${j}
83 ${ECHOCMD} "${LCTL} <<-EOF
85 --threads ${threads} 0 ${echodev} test_create \
86 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
91 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
92 REFORMAT="yes" cleanup_and_setup_lustre
95 lfsck_create_nfiles() {
100 local ldir="/test-${lbase}"
105 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
106 local usize=$((count / NTHREADS))
107 [ ${usize} -eq 0 ] && break
108 local tdir=${ldir}-${cycle}-
110 echo "[cycle: ${cycle}] [threads: ${threads}]"\
111 "[files: ${count}] [basedir: ${tdir}]"
112 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
113 error "Fail to start MDS!"
114 #define OBD_FAIL_FID_IGIF 0x1504
115 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
121 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
122 stop ${SINGLEMDS} || error "Fail to stop MDS!"
124 total=$((total - usize * NTHREADS))
125 [ ${total} -eq 0 ] && break
126 lbase=$((lbase + usize))
131 format_start_mgs () {
132 do_rpc_nodes $(facet_active_host mgs) load_modules_local
134 start mgs $(mgsdevname) $MGS_MOUNT_OPTS
142 combined_mgs_mds || format_start_mgs
143 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
144 format_mdt $(facet_number $SINGLEMDS)
146 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
147 local nfiles=$((i - BCOUNT))
149 echo "+++ start to create for ${i} files set at: $(date) +++"
150 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
151 error "Fail to create files!"
152 echo "+++ end to create for ${i} files set at: $(date) +++"
155 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
156 error "Fail to start MDS!"
158 echo "start lfsck_namespace for ${i} files set at: $(date)"
159 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
162 local STATUS=$($SHOW_NAMESPACE |
163 awk '/^status/ { print $2 }')
164 [ "$STATUS" == "completed" ] && break
165 sleep 3 # check status every 3 seconds
168 echo "end lfsck_namespace for ${i} files set at: $(date)"
169 SPEED=$($SHOW_NAMESPACE |
170 awk '/^average_speed_phase1/ { print $2 }')
171 echo "lfsck_namespace speed is ${SPEED}/sec"
172 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
174 combined_mgs_mds || stop mgs
176 run_test 0 "lfsck namespace performance (routine case) without load"
179 [ "$mds1_FSTYPE" != ldiskfs ] &&
180 skip_env "not implemented for ZFS"
186 combined_mgs_mds || format_start_mgs
187 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
188 format_mdt $(facet_number $SINGLEMDS)
190 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
191 i = $((i * FACTOR)))); do
192 local nfiles=$((i - BCOUNT))
194 echo "+++ start to create for ${i} files set at: $(date) +++"
195 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
196 error "Fail to create files!"
197 echo "+++ end to create for ${i} files set at: $(date) +++"
200 local stime=$(date +%s)
201 echo "backup/restore ${i} files start at: $(date)"
202 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
203 echo "backup/restore ${i} files end at: $(date)"
204 local etime=$(date +%s)
205 local delta=$((etime - stime))
206 [ $delta -gt 0 ] || delta=1
207 echo "backup/restore ${i} files used ${delta} seconds"
208 echo "backup/restore speed is $((i / delta))/sec"
210 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
211 error "Fail to start MDS!"
213 echo "start lfsck_namespace for ${i} files set at: $(date)"
214 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
217 local STATUS=$($SHOW_NAMESPACE |
218 awk '/^status/ { print $2 }')
219 [ "$STATUS" == "completed" ] && break
220 sleep 3 # check status every 3 seconds
223 echo "end lfsck_namespace for ${i} files set at: $(date)"
224 local SPEED=$($SHOW_NAMESPACE |
225 awk '/^average_speed_phase1/ { print $2 }')
226 echo "lfsck_namespace speed is ${SPEED}/sec"
227 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
229 combined_mgs_mds || stop mgs
231 run_test 1 "lfsck namespace performance (backup/restore) without load"
236 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
237 i = $((i * FACTOR)))); do
239 combined_mgs_mds || format_start_mgs
240 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
241 format_mdt $(facet_number $SINGLEMDS)
243 echo "+++ start to create for ${i} files set at: $(date) +++"
244 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
245 error "Fail to create files!"
246 echo "+++ end to create for ${i} files set at: $(date) +++"
248 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
249 error "Fail to start MDS!"
251 echo "start lfsck_namespace for ${i} files set at: $(date)"
252 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
255 local STATUS=$($SHOW_NAMESPACE |
256 awk '/^status/ { print $2 }')
257 [ "$STATUS" == "completed" ] && break
258 sleep 3 # check status every 3 seconds
261 echo "end lfsck_namespace for ${i} files set at: $(date)"
262 local SPEED=$($SHOW_NAMESPACE |
263 awk '/^average_speed_phase1/ { print $2 }')
264 echo "lfsck_namespace speed is ${SPEED}/sec"
265 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
267 combined_mgs_mds || stop mgs
269 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
272 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
274 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
279 combined_mgs_mds || format_start_mgs
280 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
281 format_mdt $(facet_number $SINGLEMDS)
283 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
284 local nfiles=$((i - BCOUNT))
286 echo "+++ start to create for ${i} files set at: $(date) +++"
287 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
288 error "Fail to create files!"
289 echo "+++ end to create for ${i} files set at: $(date) +++"
293 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
294 error "Fail to start MDS!"
296 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
297 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
300 local STATUS=$($SHOW_NAMESPACE |
301 awk '/^status/ { print $2 }')
302 [ "$STATUS" == "completed" ] && break
303 sleep 3 # check status every 3 seconds
306 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
307 local FULL_SPEED=$($SHOW_NAMESPACE |
308 awk '/^average_speed_phase1/ { print $2 }')
309 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
310 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
311 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
314 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
315 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
316 error "Fail to start MDS!"
318 $STOP_LFSCK > /dev/null 2>&1
319 echo "start lfsck_namespace with speed ${j} at: $(date)"
320 $START_NAMESPACE --reset -s ${j} ||
321 error "Fail to start lfsck_namespace with speed ${j}!"
322 # lfsck_namespace will be paused when MDS stop,
323 # and will be restarted automatically when mount up again.
324 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
326 local nfiles=$(((i - BCOUNT) / 2))
328 echo "+++ start to create for ${i} files set at: $(date) +++"
329 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
330 error "Fail to create files!"
331 echo "+++ end to create for ${i} files set at: $(date) +++"
336 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
337 error "Fail to start MDS!"
339 $STOP_LFSCK /dev/null 2>&1
340 echo "start lfsck_namespace with full speed at: $(date)"
341 $START_NAMESPACE --reset -s 0 ||
342 error "Fail to start lfsck_namespace with full speed!"
343 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
345 local nfiles=$(((i - BCOUNT) / 2))
347 echo "+++ start to create for ${i} files set at: $(date) +++"
348 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
349 error "Fail to create files!"
350 echo "+++ end to create for ${i} files set at: $(date) +++"
351 combined_mgs_mds || stop mgs
353 run_test 3 "lfsck namespace impact on create performance"
359 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
363 echo "***** Start layout LFSCK on all devices at: $(date) *****"
364 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
366 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
367 mdd.${MDT_DEV}.lfsck_layout |
368 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
372 echo "***** End layout LFSCK on all devices at: $(date) *****"
374 for n in $(seq $MDSCOUNT); do
377 local SPEED=$(show_layout ${n} |
378 awk '/^average_speed_phase1/ { print $2 }')
380 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
388 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
389 local basedir="$mntpt/$tdir/$idx1/$idx2"
392 error_noexit "(11) Fail to mkdir $mntpt"
396 mount_client $mntpt || {
397 error_noexit "(12) Fail to mount $mntpt"
403 error_noexit "(13) Fail to mkdir $basedir"
407 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
408 createmany -o ${basedir}/f $UNIT || {
410 error_noexit "(14) Fail to gen $UNIT files under $basedir"
413 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
421 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
422 for ((k = 0; k < $MDSCOUNT; k++)); do
423 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
425 for ((l = 1; l <= $cnt; l++)); do
426 layout_gen_one ${k} ${l} &
431 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
435 local saved_mdscount=$MDSCOUNT
436 local saved_ostcount=$OSTCOUNT
439 stopall > /dev/null || error "(1) Fail to stopall"
441 LFSCKDIR="$DIR/$tdir"
443 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
446 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
449 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
451 formatall > /dev/null ||
452 error "(2) Fail to formatall, subdirs=${j}"
455 setupall > /dev/null ||
456 error "(3) Fail to setupall, subdirs=${j}"
459 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
461 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
462 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
465 layout_gen_set ${j} || RC=$?
467 error "(6) generate set $RC, subdirs=${j}"
470 layout_test_one || RC=$?
472 error "(7) LFSCK failed with $RC, subdirs=${j}"
476 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
479 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
482 MDSCOUNT=$saved_mdscount
483 OSTCOUNT=$saved_ostcount
489 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
492 echo "Inject failure stub to simulate dangling reference"
493 #define OBD_FAIL_LFSCK_DANGLING 0x1610
494 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
498 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
501 local saved_mdscount=$MDSCOUNT
504 stopall > /dev/null || error "(1) Fail to stopall"
506 LFSCKDIR="$DIR/$tdir"
507 for ((i = 1; i <= $saved_mdscount; i++)); do
510 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
513 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
515 formatall > /dev/null ||
516 error "(2) Fail to formatall, subdirs=${j}"
519 setupall > /dev/null ||
520 error "(3) Fail to setupall, subdirs=${j}"
523 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
525 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
526 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
529 layout_gen_set ${j} || RC=$?
531 error "(6) generate set $RC, subdirs=${j}"
534 layout_test_one || RC=$?
536 error "(7) LFSCK failed with $RC, subdirs=${j}"
540 stopall > /dev/null || error "(8) Fail to stopall"
543 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
546 MDSCOUNT=$saved_mdscount
552 run_test 5a "lfsck layout performance (routine case) without load for DNE"
555 echo "Inject failure stub to simulate dangling reference"
556 #define OBD_FAIL_LFSCK_DANGLING 0x1610
557 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
561 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
563 lfsck_fast_create() {
567 local ldir="/test-${lbase}"
572 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
573 local usize=$((count / NTHREADS))
574 [ ${usize} -eq 0 ] && break
575 local tdir=${ldir}-${cycle}-
577 echo "[cycle: $cycle] [threads: $threads]"\
578 "[files: $count] [basedir: $tdir]"
582 total=$((total - usize * NTHREADS))
583 [ $total -eq 0 ] && break
584 lbase=$((lbase + usize))
589 lfsck_detach_error() {
595 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
598 stopall > /dev/null || error "(1) Fail to stopall"
600 local saved_mdscount=$MDSCOUNT
602 LFSCKDIR="$DIR/$tdir"
605 formatall > /dev/null || error "(2) Fail to formatall"
608 setupall > /dev/null || error "(3) Fail to setupall"
610 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
612 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
613 error "(5) Fail to setstripe on $LFSCKDIR"
616 layout_gen_set $TOTSUBDIR || RC=$?
618 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
621 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
622 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
623 error "(7) Fail to start layout LFSCK"
625 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
626 mdd.${MDT_DEV}.lfsck_layout |
627 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
629 error "(8) layout LFSCK cannot finished in time"
631 echo "***** End layout LFSCK on single MDS at: $(date) *****"
633 local SPEED=$(show_layout 1 |
634 awk '/^average_speed_phase1/ { print $2 }')
635 echo "lfsck_layout full_speed is $SPEED objs/sec"
637 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
638 local nfiles=$((inc_count / 2))
641 for ((m = 0, n = $INCFACTOR; n < 100;
642 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
643 local sl=$((SPEED * n / 100))
645 $STOP_LFSCK > /dev/null 2>&1
647 echo "start lfsck_layout with speed ${sl} at: $(date)"
648 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
650 "(9) Fail to start lfsck_layout with speed ${sl}"
652 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
653 lfsck_fast_create $nfiles ${m} $NTHREADS ||
654 lfsck_detach_error "(10) Fail to create files"
655 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
658 $STOP_LFSCK > /dev/null 2>&1
660 echo "start lfsck_layout with full speed at: $(date)"
661 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
663 "(11) Fail to start lfsck_layout with full speed"
665 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
666 lfsck_fast_create $nfiles ${m} $NTHREADS ||
667 lfsck_detach_error "(12) Fail to create files"
668 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
671 $STOP_LFSCK > /dev/null 2>&1
673 echo "create without lfsck_layout run back-ground"
674 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
675 lfsck_fast_create $nfiles ${m} $NTHREADS ||
676 lfsck_detach_error "(13) Fail to create files"
677 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
682 stopall > /dev/null || error "(14) Fail to stopall"
684 MDSCOUNT=$saved_mdscount
686 run_test 6 "lfsck layout impact on create performance"
692 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
695 namespace_test_one() {
696 echo "***** Start namespace LFSCK on all devices at: $(date) *****"
697 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
699 for n in $(seq $MDSCOUNT); do
700 wait_update_facet mds${n} "$LCTL get_param -n \
701 mdd.$(facet_svc mds${n}).lfsck_namespace |
702 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
707 echo "***** End namespace LFSCK on all devices at: $(date) *****"
709 for n in $(seq $MDSCOUNT); do
712 local SPEED=$(show_namespace ${n} |
713 awk '/^average_speed_total/ { print $2 }')
715 echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
720 namespace_gen_one() {
723 local idx3=$(((idx1 + 1) % MDSCOUNT))
724 local base_mntpt="/mnt/lustre_lfsck_${idx1}"
725 local show_dir="$LFSCKDIR/${idx1}/${idx2}"
726 local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
728 mkdir $show_dir || return 20
730 local count=$((UNIT * 78 / 100)) # 78% regular files
731 local sub_count=$((count / NTHREADS))
732 echo "Creating $count regular files under $show_dir at: $(date)"
733 for ((m = 0; m < $NTHREADS; m++)); do
734 local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
736 createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
740 error_noexit "(21) Fail to gen regular files under $show_dir"
744 count=$((UNIT * 3 / 100)) # 3% local sub-dirs
745 echo "Creating $count local sub-dirs under $show_dir at: $(date)"
746 createmany -d $work_dir/d_l_ $count > /dev/null || {
747 error_noexit "(22) Fail to gen local sub-dir under $show_dir"
751 # 3% * 5 = 15% regular files under local sub-dirs
752 echo "Creating 5 regular files under each local sub-dir at: $(date)"
753 for ((m = 0; m < $count; m++)); do
754 createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
756 "(23) Fail to gen regular under $work_dir/d_l_${m}"
761 count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
762 echo "Creating $count multiple hard-links under $show_dir at: $(date)"
763 for ((m = 0; m < $count; m++)); do
764 ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
766 "(24) Fail to hardlink to $work_dir/f_0_${m}"
771 count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
772 echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
773 "files under each remote sub-dir at: $(date)"
774 for ((m = 0; m < $count; m++)); do
775 $LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
777 "(25) Fail to remote mkdir $work_dir/d_r_${m}"
781 # 0.3% * 4 = 1.2% regular files under remote sub-dirs
782 createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
784 "(26) Fail to gen regular under $work_dir/d_r_${m}"
789 # 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
790 count=$((UNIT * 3 / 1000))
791 echo "Creating $count 2-striped sub-dirs under $show_dir," \
792 "and 4 regular files under each striped sub-dir at: $(date)"
793 for ((m = 0; m < $count; m++)); do
794 $LFS setdirstripe -i ${idx1} -c 2 -H all_char \
795 $work_dir/d_s_${m} || {
797 "(27) Fail to make striped-dir $work_dir/d_s_${m}"
801 # 0.3% * 4 = 1.2% regular files under striped sub-dirs
802 createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
804 "(28) Fail to gen regular under $work_dir/d_s_${m}"
810 namespace_gen_mdt() {
813 local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
815 $LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
817 for ((m = 0; m < $NTHREADS; m++)); do
818 local mntpt="${base_mntpt}_${m}"
821 umount ${base_mntpt}_*
822 error_noexit "(11) Fail to mkdir $mntpt"
826 mount_client $mntpt || {
827 umount ${base_mntpt}_*
828 error_noexit "(12) Fail to mount $mntpt"
833 for ((l = 0; l < $dir_cnt; l++)); do
834 namespace_gen_one ${mdt_idx} ${l}
837 umount ${base_mntpt}_*
840 namespace_gen_set() {
844 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
845 for ((k = 0; k < $MDSCOUNT; k++)); do
846 namespace_gen_mdt ${k} ${cnt} &
849 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
854 local saved_mdscount=$MDSCOUNT
856 [ $MDSCOUNT -le 8 ] ||
857 error "Too much MDT, test data set on each MDT may be unbalance"
860 stopall > /dev/null || error "(1) Fail to stopall"
862 LFSCKDIR="$DIR/$tdir"
863 for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
866 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
869 for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
870 j = $((j + MINSUBDIR)))); do
872 formatall > /dev/null ||
873 error "(2) Fail to formatall, subdirs=${j}"
876 setupall > /dev/null ||
877 error "(3) Fail to setupall, subdirs=${j}"
880 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
882 $LFS setstripe -c 1 -i -1 $LFSCKDIR ||
883 error "(5) Fail to setstripe on $LFSCKDIR"
885 do_nodes $(comma_list $(mdts_nodes)) \
886 $LCTL set_param fail_loc=$local_loc
889 namespace_gen_set ${j} || RC=$?
891 error "(6) generate set $RC, subdirs=${j}"
894 namespace_test_one || RC=$?
896 error "(7) LFSCK failed with $RC, subdirs=${j}"
898 do_nodes $(comma_list $(mdts_nodes)) \
899 $LCTL set_param fail_loc=0
903 stopall > /dev/null || error "(8) Fail to stopall"
906 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
909 MDSCOUNT=$saved_mdscount
915 run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
918 echo "Inject failure stub to simulate the case of lost linkEA"
919 #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
922 run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
925 echo "Inject failure stub to simulate the case of bad FID-in-dirent"
926 #define OBD_FAIL_FID_INDIR 0x1501
929 run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
932 [ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
934 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
937 stopall > /dev/null || error "(1) Fail to stopall"
939 local saved_mdscount=$MDSCOUNT
941 LFSCKDIR="$DIR/$tdir"
944 formatall > /dev/null || error "(2) Fail to formatall"
947 setupall > /dev/null || error "(3) Fail to setupall"
949 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
951 $LFS setstripe -c 1 -i 0 $LFSCKDIR ||
952 error "(5) Fail to setstripe on $LFSCKDIR"
955 namespace_gen_set $TOTSUBDIR || RC=$?
957 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
960 echo "***** Start namespace LFSCK at: $(date) *****"
961 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
962 error "(7) Fail to start namespace LFSCK"
964 for n in $(seq $MDSCOUNT); do
965 wait_update_facet mds${n} "$LCTL get_param -n \
966 mdd.$(facet_svc mds${n}).lfsck_namespace |
967 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
969 error "(8) namespace LFSCK cannot finished in time"
972 echo "***** End namespace LFSCK at: $(date) *****"
974 local SPEED=$(show_namespace 1 |
975 awk '/^average_speed_phase1/ { print $2 }')
976 echo "lfsck_namespace full_speed is $SPEED objs/sec"
979 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
980 local nfiles=$((inc_count / 2))
985 local stime=$(date +%s)
986 lfsck_fast_create $nfiles ${m} $NTHREADS ||
987 lfsck_detach_error "(9) Fail to create files"
988 local etime=$(date +%s)
989 echo "created $nfiles without lfsck_namespace run back-ground used" \
990 "$((etime - stime)) seconds"
993 for ((m = nfiles, n = $INCFACTOR; n < 100;
994 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
995 local sl=$((SPEED * n / 100))
997 $STOP_LFSCK > /dev/null 2>&1
998 echo "start lfsck_namespace with speed ${sl} at: $(date)"
999 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
1000 lfsck_detach_error \
1001 "(10) Fail to start lfsck_namespace with speed ${sl}"
1004 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1005 lfsck_detach_error "(11) Fail to create files"
1007 echo "created $nfiles with namespace LFSCK run with the" \
1008 "speed limit of ${n}% of full speed used" \
1009 "$((etime - stime)) seconds"
1013 $STOP_LFSCK > /dev/null 2>&1
1014 echo "start lfsck_namespace with full speed at: $(date)"
1015 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
1016 lfsck_detach_error \
1017 "(12) Fail to start lfsck_namespace with full speed"
1020 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1021 lfsck_detach_error "(13) Fail to create files"
1023 echo "created $nfiles with namespace LFSCK run with full speed used" \
1024 "$((etime - stime)) seconds"
1027 $STOP_LFSCK > /dev/null 2>&1
1032 stopall > /dev/null || error "(14) Fail to stopall"
1034 MDSCOUNT=$saved_mdscount
1036 run_test 8 "lfsck namespace impact on create performance"
1038 # cleanup the system at last
1041 check_and_cleanup_lustre