6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 require_dsh_mds || exit 0
17 require_dsh_ost || exit 0
20 skip "skip lfsck performance test under non-SLOW mode" && exit 0
22 NTHREADS=${NTHREADS:-0}
24 MINCOUNT=${MINCOUNT:-4096}
25 MAXCOUNT=${MAXCOUNT:-8192}
26 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
27 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
28 BASE_COUNT=${BASE_COUNT:-8192}
30 INCFACTOR=${INCFACTOR:-25} #percent
31 MINSUBDIR=${MINSUBDIR:-1}
32 MAXSUBDIR=${MAXSUBDIR:-2}
33 TOTSUBDIR=${TOTSUBDIR:-2}
36 RCMD="do_facet ${SINGLEMDS}"
37 RLCTL="${RCMD} ${LCTL}"
38 MDT_DEV="${FSNAME}-MDT0000"
39 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
40 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
42 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
44 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
46 if [ ${NTHREADS} -eq 0 ]; then
47 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
48 NTHREADS=$((CPUCORE * 2))
52 ${RCMD} "modprobe obdecho"
54 ${ECHOCMD} "${LCTL} <<-EOF
55 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
61 ${ECHOCMD} "${LCTL} <<-EOF
69 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
72 ${ECHOCMD} "${LCTL} <<-EOF
77 for ((j = 1; j < ${threads}; j++)); do
78 ${ECHOCMD} "${LCTL} <<-EOF
80 test_mkdir ${tdir}${j}
84 ${ECHOCMD} "${LCTL} <<-EOF
86 --threads ${threads} 0 ${echodev} test_create \
87 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
92 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
96 lfsck_create_nfiles() {
101 local ldir="/test-${lbase}"
106 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
107 local usize=$((count / NTHREADS))
108 [ ${usize} -eq 0 ] && break
109 local tdir=${ldir}-${cycle}-
111 echo "[cycle: ${cycle}] [threads: ${threads}]"\
112 "[files: ${count}] [basedir: ${tdir}]"
113 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
114 error "Fail to start MDS!"
115 #define OBD_FAIL_FID_IGIF 0x1504
116 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
122 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
123 stop ${SINGLEMDS} || error "Fail to stop MDS!"
125 total=$((total - usize * NTHREADS))
126 [ ${total} -eq 0 ] && break
127 lbase=$((lbase + usize))
134 format_start_mgs () {
135 do_rpc_nodes $(facet_active_host mgs) load_modules_local
137 start mgs $(mgsdevname) $MGS_MOUNT_OPTS
145 combined_mgs_mds || format_start_mgs
146 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
147 format_mdt $(facet_number $SINGLEMDS)
149 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
150 local nfiles=$((i - BCOUNT))
152 echo "+++ start to create for ${i} files set at: $(date) +++"
153 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
154 error "Fail to create files!"
155 echo "+++ end to create for ${i} files set at: $(date) +++"
158 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
159 error "Fail to start MDS!"
161 echo "start lfsck_namespace for ${i} files set at: $(date)"
162 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
165 local STATUS=$($SHOW_NAMESPACE |
166 awk '/^status/ { print $2 }')
167 [ "$STATUS" == "completed" ] && break
168 sleep 3 # check status every 3 seconds
171 echo "end lfsck_namespace for ${i} files set at: $(date)"
172 SPEED=$($SHOW_NAMESPACE |
173 awk '/^average_speed_phase1/ { print $2 }')
174 echo "lfsck_namespace speed is ${SPEED}/sec"
175 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
177 combined_mgs_mds || stop mgs
179 run_test 0 "lfsck namespace performance (routine case) without load"
182 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
183 skip "not implemented for ZFS" && return
189 combined_mgs_mds || format_start_mgs
190 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
191 format_mdt $(facet_number $SINGLEMDS)
193 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
194 i = $((i * FACTOR)))); do
195 local nfiles=$((i - BCOUNT))
197 echo "+++ start to create for ${i} files set at: $(date) +++"
198 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
199 error "Fail to create files!"
200 echo "+++ end to create for ${i} files set at: $(date) +++"
203 local stime=$(date +%s)
204 echo "backup/restore ${i} files start at: $(date)"
205 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
206 echo "backup/restore ${i} files end at: $(date)"
207 local etime=$(date +%s)
208 local delta=$((etime - stime))
209 [ $delta -gt 0 ] || delta=1
210 echo "backup/restore ${i} files used ${delta} seconds"
211 echo "backup/restore speed is $((i / delta))/sec"
213 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
214 error "Fail to start MDS!"
216 echo "start lfsck_namespace for ${i} files set at: $(date)"
217 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
220 local STATUS=$($SHOW_NAMESPACE |
221 awk '/^status/ { print $2 }')
222 [ "$STATUS" == "completed" ] && break
223 sleep 3 # check status every 3 seconds
226 echo "end lfsck_namespace for ${i} files set at: $(date)"
227 local SPEED=$($SHOW_NAMESPACE |
228 awk '/^average_speed_phase1/ { print $2 }')
229 echo "lfsck_namespace speed is ${SPEED}/sec"
230 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
232 combined_mgs_mds || stop mgs
234 run_test 1 "lfsck namespace performance (backup/restore) without load"
239 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
240 i = $((i * FACTOR)))); do
242 combined_mgs_mds || format_start_mgs
243 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
244 format_mdt $(facet_number $SINGLEMDS)
246 echo "+++ start to create for ${i} files set at: $(date) +++"
247 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
248 error "Fail to create files!"
249 echo "+++ end to create for ${i} files set at: $(date) +++"
251 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
252 error "Fail to start MDS!"
254 echo "start lfsck_namespace for ${i} files set at: $(date)"
255 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
258 local STATUS=$($SHOW_NAMESPACE |
259 awk '/^status/ { print $2 }')
260 [ "$STATUS" == "completed" ] && break
261 sleep 3 # check status every 3 seconds
264 echo "end lfsck_namespace for ${i} files set at: $(date)"
265 local SPEED=$($SHOW_NAMESPACE |
266 awk '/^average_speed_phase1/ { print $2 }')
267 echo "lfsck_namespace speed is ${SPEED}/sec"
268 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
270 combined_mgs_mds || stop mgs
272 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
275 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
277 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
282 combined_mgs_mds || format_start_mgs
283 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
284 format_mdt $(facet_number $SINGLEMDS)
286 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
287 local nfiles=$((i - BCOUNT))
289 echo "+++ start to create for ${i} files set at: $(date) +++"
290 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
291 error "Fail to create files!"
292 echo "+++ end to create for ${i} files set at: $(date) +++"
296 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
297 error "Fail to start MDS!"
299 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
300 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
303 local STATUS=$($SHOW_NAMESPACE |
304 awk '/^status/ { print $2 }')
305 [ "$STATUS" == "completed" ] && break
306 sleep 3 # check status every 3 seconds
309 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
310 local FULL_SPEED=$($SHOW_NAMESPACE |
311 awk '/^average_speed_phase1/ { print $2 }')
312 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
313 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
314 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
317 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
318 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
319 error "Fail to start MDS!"
321 $STOP_LFSCK > /dev/null 2>&1
322 echo "start lfsck_namespace with speed ${j} at: $(date)"
323 $START_NAMESPACE --reset -s ${j} ||
324 error "Fail to start lfsck_namespace with speed ${j}!"
325 # lfsck_namespace will be paused when MDS stop,
326 # and will be restarted automatically when mount up again.
327 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
329 local nfiles=$(((i - BCOUNT) / 2))
331 echo "+++ start to create for ${i} files set at: $(date) +++"
332 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
333 error "Fail to create files!"
334 echo "+++ end to create for ${i} files set at: $(date) +++"
339 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
340 error "Fail to start MDS!"
342 $STOP_LFSCK /dev/null 2>&1
343 echo "start lfsck_namespace with full speed at: $(date)"
344 $START_NAMESPACE --reset -s 0 ||
345 error "Fail to start lfsck_namespace with full speed!"
346 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
348 local nfiles=$(((i - BCOUNT) / 2))
350 echo "+++ start to create for ${i} files set at: $(date) +++"
351 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
352 error "Fail to create files!"
353 echo "+++ end to create for ${i} files set at: $(date) +++"
354 combined_mgs_mds || stop mgs
356 run_test 3 "lfsck namespace impact on create performance"
362 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
366 echo "***** Start layout LFSCK on all devices at: $(date) *****"
367 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
369 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
370 mdd.${MDT_DEV}.lfsck_layout |
371 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
375 echo "***** End layout LFSCK on all devices at: $(date) *****"
377 for n in $(seq $MDSCOUNT); do
380 local SPEED=$(show_layout ${n} |
381 awk '/^average_speed_phase1/ { print $2 }')
383 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
391 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
392 local basedir="$mntpt/$tdir/$idx1/$idx2"
395 error_noexit "(11) Fail to mkdir $mntpt"
399 mount_client $mntpt || {
400 error_noexit "(12) Fail to mount $mntpt"
406 error_noexit "(13) Fail to mkdir $basedir"
410 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
411 createmany -o ${basedir}/f $UNIT || {
413 error_noexit "(14) Fail to gen $UNIT files under $basedir"
416 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
424 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
425 for ((k = 0; k < $MDSCOUNT; k++)); do
426 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
428 for ((l = 1; l <= $cnt; l++)); do
429 layout_gen_one ${k} ${l} &
434 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
438 local saved_mdscount=$MDSCOUNT
439 local saved_ostcount=$OSTCOUNT
442 stopall > /dev/null || error "(1) Fail to stopall"
444 LFSCKDIR="$DIR/$tdir"
446 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
449 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
452 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
454 formatall > /dev/null ||
455 error "(2) Fail to formatall, subdirs=${j}"
458 setupall > /dev/null ||
459 error "(3) Fail to setupall, subdirs=${j}"
462 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
464 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
465 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
468 layout_gen_set ${j} || RC=$?
470 error "(6) generate set $RC, subdirs=${j}"
473 layout_test_one || RC=$?
475 error "(7) LFSCK failed with $RC, subdirs=${j}"
479 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
482 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
485 MDSCOUNT=$saved_mdscount
486 OSTCOUNT=$saved_ostcount
492 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
495 echo "Inject failure stub to simulate dangling reference"
496 #define OBD_FAIL_LFSCK_DANGLING 0x1610
497 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
501 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
504 local saved_mdscount=$MDSCOUNT
507 stopall > /dev/null || error "(1) Fail to stopall"
509 LFSCKDIR="$DIR/$tdir"
510 for ((i = 1; i <= $saved_mdscount; i++)); do
513 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
516 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
518 formatall > /dev/null ||
519 error "(2) Fail to formatall, subdirs=${j}"
522 setupall > /dev/null ||
523 error "(3) Fail to setupall, subdirs=${j}"
526 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
528 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
529 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
532 layout_gen_set ${j} || RC=$?
534 error "(6) generate set $RC, subdirs=${j}"
537 layout_test_one || RC=$?
539 error "(7) LFSCK failed with $RC, subdirs=${j}"
543 stopall > /dev/null || error "(8) Fail to stopall"
546 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
549 MDSCOUNT=$saved_mdscount
555 run_test 5a "lfsck layout performance (routine case) without load for DNE"
558 echo "Inject failure stub to simulate dangling reference"
559 #define OBD_FAIL_LFSCK_DANGLING 0x1610
560 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
564 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
566 lfsck_fast_create() {
570 local ldir="/test-${lbase}"
575 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
576 local usize=$((count / NTHREADS))
577 [ ${usize} -eq 0 ] && break
578 local tdir=${ldir}-${cycle}-
580 echo "[cycle: $cycle] [threads: $threads]"\
581 "[files: $count] [basedir: $tdir]"
585 total=$((total - usize * NTHREADS))
586 [ $total -eq 0 ] && break
587 lbase=$((lbase + usize))
592 lfsck_detach_error() {
598 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
601 stopall > /dev/null || error "(1) Fail to stopall"
603 local saved_mdscount=$MDSCOUNT
605 LFSCKDIR="$DIR/$tdir"
608 formatall > /dev/null || error "(2) Fail to formatall"
611 setupall > /dev/null || error "(3) Fail to setupall"
613 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
615 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
616 error "(5) Fail to setstripe on $LFSCKDIR"
619 layout_gen_set $TOTSUBDIR || RC=$?
621 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
624 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
625 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
626 error "(7) Fail to start layout LFSCK"
628 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
629 mdd.${MDT_DEV}.lfsck_layout |
630 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
632 error "(8) layout LFSCK cannot finished in time"
634 echo "***** End layout LFSCK on single MDS at: $(date) *****"
636 local SPEED=$(show_layout 1 |
637 awk '/^average_speed_phase1/ { print $2 }')
638 echo "lfsck_layout full_speed is $SPEED objs/sec"
640 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
641 local nfiles=$((inc_count / 2))
644 for ((m = 0, n = $INCFACTOR; n < 100;
645 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
646 local sl=$((SPEED * n / 100))
648 $STOP_LFSCK > /dev/null 2>&1
650 echo "start lfsck_layout with speed ${sl} at: $(date)"
651 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
653 "(9) Fail to start lfsck_layout with speed ${sl}"
655 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
656 lfsck_fast_create $nfiles ${m} $NTHREADS ||
657 lfsck_detach_error "(10) Fail to create files"
658 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
661 $STOP_LFSCK > /dev/null 2>&1
663 echo "start lfsck_layout with full speed at: $(date)"
664 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
666 "(11) Fail to start lfsck_layout with full speed"
668 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
669 lfsck_fast_create $nfiles ${m} $NTHREADS ||
670 lfsck_detach_error "(12) Fail to create files"
671 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
674 $STOP_LFSCK > /dev/null 2>&1
676 echo "create without lfsck_layout run back-ground"
677 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
678 lfsck_fast_create $nfiles ${m} $NTHREADS ||
679 lfsck_detach_error "(13) Fail to create files"
680 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
685 stopall > /dev/null || error "(14) Fail to stopall"
687 MDSCOUNT=$saved_mdscount
689 run_test 6 "lfsck layout impact on create performance"
695 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
698 namespace_test_one() {
699 echo "***** Start namespace LFSCK on all devices at: $(date) *****"
700 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
702 for n in $(seq $MDSCOUNT); do
703 wait_update_facet mds${n} "$LCTL get_param -n \
704 mdd.$(facet_svc mds${n}).lfsck_namespace |
705 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
710 echo "***** End namespace LFSCK on all devices at: $(date) *****"
712 for n in $(seq $MDSCOUNT); do
715 local SPEED=$(show_namespace ${n} |
716 awk '/^average_speed_total/ { print $2 }')
718 echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
723 namespace_gen_one() {
726 local idx3=$(((idx1 + 1) % MDSCOUNT))
727 local base_mntpt="/mnt/lustre_lfsck_${idx1}"
728 local show_dir="$LFSCKDIR/${idx1}/${idx2}"
729 local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
731 mkdir $show_dir || return 20
733 local count=$((UNIT * 78 / 100)) # 78% regular files
734 local sub_count=$((count / NTHREADS))
735 echo "Creating $count regular files under $show_dir at: $(date)"
736 for ((m = 0; m < $NTHREADS; m++)); do
737 local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
739 createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
743 error_noexit "(21) Fail to gen regular files under $show_dir"
747 count=$((UNIT * 3 / 100)) # 3% local sub-dirs
748 echo "Creating $count local sub-dirs under $show_dir at: $(date)"
749 createmany -d $work_dir/d_l_ $count > /dev/null || {
750 error_noexit "(22) Fail to gen local sub-dir under $show_dir"
754 # 3% * 5 = 15% regular files under local sub-dirs
755 echo "Creating 5 regular files under each local sub-dir at: $(date)"
756 for ((m = 0; m < $count; m++)); do
757 createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
759 "(23) Fail to gen regular under $work_dir/d_l_${m}"
764 count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
765 echo "Creating $count multiple hard-links under $show_dir at: $(date)"
766 for ((m = 0; m < $count; m++)); do
767 ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
769 "(24) Fail to hardlink to $work_dir/f_0_${m}"
774 count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
775 echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
776 "files under each remote sub-dir at: $(date)"
777 for ((m = 0; m < $count; m++)); do
778 $LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
780 "(25) Fail to remote mkdir $work_dir/d_r_${m}"
784 # 0.3% * 4 = 1.2% regular files under remote sub-dirs
785 createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
787 "(26) Fail to gen regular under $work_dir/d_r_${m}"
792 # 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
793 count=$((UNIT * 3 / 1000))
794 echo "Creating $count 2-striped sub-dirs under $show_dir," \
795 "and 4 regular files under each striped sub-dir at: $(date)"
796 for ((m = 0; m < $count; m++)); do
797 $LFS setdirstripe -i ${idx1} -c 2 -H all_char \
798 $work_dir/d_s_${m} || {
800 "(27) Fail to make striped-dir $work_dir/d_s_${m}"
804 # 0.3% * 4 = 1.2% regular files under striped sub-dirs
805 createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
807 "(28) Fail to gen regular under $work_dir/d_s_${m}"
813 namespace_gen_mdt() {
816 local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
818 $LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
820 for ((m = 0; m < $NTHREADS; m++)); do
821 local mntpt="${base_mntpt}_${m}"
824 umount ${base_mntpt}_*
825 error_noexit "(11) Fail to mkdir $mntpt"
829 mount_client $mntpt || {
830 umount ${base_mntpt}_*
831 error_noexit "(12) Fail to mount $mntpt"
836 for ((l = 0; l < $dir_cnt; l++)); do
837 namespace_gen_one ${mdt_idx} ${l}
840 umount ${base_mntpt}_*
843 namespace_gen_set() {
847 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
848 for ((k = 0; k < $MDSCOUNT; k++)); do
849 namespace_gen_mdt ${k} ${cnt} &
852 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
857 local saved_mdscount=$MDSCOUNT
859 [ $MDSCOUNT -le 8 ] ||
860 error "Too much MDT, test data set on each MDT may be unbalance"
863 stopall > /dev/null || error "(1) Fail to stopall"
865 LFSCKDIR="$DIR/$tdir"
866 for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
869 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
872 for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
873 j = $((j + MINSUBDIR)))); do
875 formatall > /dev/null ||
876 error "(2) Fail to formatall, subdirs=${j}"
879 setupall > /dev/null ||
880 error "(3) Fail to setupall, subdirs=${j}"
883 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
885 $LFS setstripe -c 1 -i -1 $LFSCKDIR ||
886 error "(5) Fail to setstripe on $LFSCKDIR"
888 do_nodes $(comma_list $(mdts_nodes)) \
889 $LCTL set_param fail_loc=$local_loc
892 namespace_gen_set ${j} || RC=$?
894 error "(6) generate set $RC, subdirs=${j}"
897 namespace_test_one || RC=$?
899 error "(7) LFSCK failed with $RC, subdirs=${j}"
901 do_nodes $(comma_list $(mdts_nodes)) \
902 $LCTL set_param fail_loc=0
906 stopall > /dev/null || error "(8) Fail to stopall"
909 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
912 MDSCOUNT=$saved_mdscount
918 run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
921 echo "Inject failure stub to simulate the case of lost linkEA"
922 #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
925 run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
928 echo "Inject failure stub to simulate the case of bad FID-in-dirent"
929 #define OBD_FAIL_FID_INDIR 0x1501
932 run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
935 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
937 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
940 stopall > /dev/null || error "(1) Fail to stopall"
942 local saved_mdscount=$MDSCOUNT
944 LFSCKDIR="$DIR/$tdir"
947 formatall > /dev/null || error "(2) Fail to formatall"
950 setupall > /dev/null || error "(3) Fail to setupall"
952 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
954 $LFS setstripe -c 1 -i 0 $LFSCKDIR ||
955 error "(5) Fail to setstripe on $LFSCKDIR"
958 namespace_gen_set $TOTSUBDIR || RC=$?
960 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
963 echo "***** Start namespace LFSCK at: $(date) *****"
964 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
965 error "(7) Fail to start namespace LFSCK"
967 for n in $(seq $MDSCOUNT); do
968 wait_update_facet mds${n} "$LCTL get_param -n \
969 mdd.$(facet_svc mds${n}).lfsck_namespace |
970 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
972 error "(8) namespace LFSCK cannot finished in time"
975 echo "***** End namespace LFSCK at: $(date) *****"
977 local SPEED=$(show_namespace 1 |
978 awk '/^average_speed_phase1/ { print $2 }')
979 echo "lfsck_namespace full_speed is $SPEED objs/sec"
982 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
983 local nfiles=$((inc_count / 2))
988 local stime=$(date +%s)
989 lfsck_fast_create $nfiles ${m} $NTHREADS ||
990 lfsck_detach_error "(9) Fail to create files"
991 local etime=$(date +%s)
992 echo "created $nfiles without lfsck_namespace run back-ground used" \
993 "$((etime - stime)) seconds"
996 for ((m = nfiles, n = $INCFACTOR; n < 100;
997 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
998 local sl=$((SPEED * n / 100))
1000 $STOP_LFSCK > /dev/null 2>&1
1001 echo "start lfsck_namespace with speed ${sl} at: $(date)"
1002 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
1003 lfsck_detach_error \
1004 "(10) Fail to start lfsck_namespace with speed ${sl}"
1007 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1008 lfsck_detach_error "(11) Fail to create files"
1010 echo "created $nfiles with namespace LFSCK run with the" \
1011 "speed limit of ${n}% of full speed used" \
1012 "$((etime - stime)) seconds"
1016 $STOP_LFSCK > /dev/null 2>&1
1017 echo "start lfsck_namespace with full speed at: $(date)"
1018 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
1019 lfsck_detach_error \
1020 "(12) Fail to start lfsck_namespace with full speed"
1023 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1024 lfsck_detach_error "(13) Fail to create files"
1026 echo "created $nfiles with namespace LFSCK run with full speed used" \
1027 "$((etime - stime)) seconds"
1030 $STOP_LFSCK > /dev/null 2>&1
1035 stopall > /dev/null || error "(14) Fail to stopall"
1037 MDSCOUNT=$saved_mdscount
1039 run_test 8 "lfsck namespace impact on create performance"
1041 # cleanup the system at last