6 ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
7 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
8 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
10 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
11 . $LUSTRE/tests/test-framework.sh
13 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
16 require_dsh_mds || exit 0
17 require_dsh_ost || exit 0
20 skip "skip lfsck performance test under non-SLOW mode" && exit 0
22 NTHREADS=${NTHREADS:-0}
24 MINCOUNT=${MINCOUNT:-4096}
25 MAXCOUNT=${MAXCOUNT:-8192}
26 MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
27 MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
28 BASE_COUNT=${BASE_COUNT:-8192}
30 INCFACTOR=${INCFACTOR:-25} #percent
31 MINSUBDIR=${MINSUBDIR:-1}
32 MAXSUBDIR=${MAXSUBDIR:-2}
33 TOTSUBDIR=${TOTSUBDIR:-2}
36 RCMD="do_facet ${SINGLEMDS}"
37 RLCTL="${RCMD} ${LCTL}"
38 MDT_DEV="${FSNAME}-MDT0000"
39 MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
40 START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
41 STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
42 SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
43 MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
44 remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
46 if [ ${NTHREADS} -eq 0 ]; then
47 CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
48 NTHREADS=$((CPUCORE * 2))
52 ${RCMD} "modprobe obdecho"
54 ${ECHOCMD} "${LCTL} <<-EOF
55 attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
61 ${ECHOCMD} "${LCTL} <<-EOF
69 local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
72 ${ECHOCMD} "${LCTL} <<-EOF
77 for ((j = 1; j < ${threads}; j++)); do
78 ${ECHOCMD} "${LCTL} <<-EOF
80 test_mkdir ${tdir}${j}
84 ${ECHOCMD} "${LCTL} <<-EOF
86 --threads ${threads} 0 ${echodev} test_create \
87 -d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
92 do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
96 lfsck_create_nfiles() {
101 local ldir="/test-${lbase}"
106 [ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
107 local usize=$((count / NTHREADS))
108 [ ${usize} -eq 0 ] && break
109 local tdir=${ldir}-${cycle}-
111 echo "[cycle: ${cycle}] [threads: ${threads}]"\
112 "[files: ${count}] [basedir: ${tdir}]"
113 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
114 error "Fail to start MDS!"
115 #define OBD_FAIL_FID_IGIF 0x1504
116 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
122 [ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
123 stop ${SINGLEMDS} || error "Fail to stop MDS!"
125 total=$((total - usize * NTHREADS))
126 [ ${total} -eq 0 ] && break
127 lbase=$((lbase + usize))
139 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
140 reformat_external_journal
141 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
142 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
143 $(mdsvdevname 1) >/dev/null || error "Fail to reformat the MDS!"
145 for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
146 local nfiles=$((i - BCOUNT))
148 echo "+++ start to create for ${i} files set at: $(date) +++"
149 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
150 error "Fail to create files!"
151 echo "+++ end to create for ${i} files set at: $(date) +++"
154 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
155 error "Fail to start MDS!"
157 echo "start lfsck_namespace for ${i} files set at: $(date)"
158 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
161 local STATUS=$($SHOW_NAMESPACE |
162 awk '/^status/ { print $2 }')
163 [ "$STATUS" == "completed" ] && break
164 sleep 3 # check status every 3 seconds
167 echo "end lfsck_namespace for ${i} files set at: $(date)"
168 SPEED=$($SHOW_NAMESPACE |
169 awk '/^average_speed_phase1/ { print $2 }')
170 echo "lfsck_namespace speed is ${SPEED}/sec"
171 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
174 run_test 0 "lfsck namespace performance (routine case) without load"
177 [ $(facet_fstype $SINGLEMDS) != ldiskfs ] &&
178 skip "not implemented for ZFS" && return
184 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
185 reformat_external_journal
186 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
187 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
188 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
190 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
191 i = $((i * FACTOR)))); do
192 local nfiles=$((i - BCOUNT))
194 echo "+++ start to create for ${i} files set at: $(date) +++"
195 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
196 error "Fail to create files!"
197 echo "+++ end to create for ${i} files set at: $(date) +++"
200 local stime=$(date +%s)
201 echo "backup/restore ${i} files start at: $(date)"
202 mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
203 echo "backup/restore ${i} files end at: $(date)"
204 local etime=$(date +%s)
205 local delta=$((etime - stime))
206 [ $delta -gt 0 ] || delta=1
207 echo "backup/restore ${i} files used ${delta} seconds"
208 echo "backup/restore speed is $((i / delta))/sec"
210 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
211 error "Fail to start MDS!"
213 echo "start lfsck_namespace for ${i} files set at: $(date)"
214 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
217 local STATUS=$($SHOW_NAMESPACE |
218 awk '/^status/ { print $2 }')
219 [ "$STATUS" == "completed" ] && break
220 sleep 3 # check status every 3 seconds
223 echo "end lfsck_namespace for ${i} files set at: $(date)"
224 local SPEED=$($SHOW_NAMESPACE |
225 awk '/^average_speed_phase1/ { print $2 }')
226 echo "lfsck_namespace speed is ${SPEED}/sec"
227 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
230 run_test 1 "lfsck namespace performance (backup/restore) without load"
235 for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
236 i = $((i * FACTOR)))); do
238 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
239 reformat_external_journal
240 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) \
241 --backfstype $(facet_fstype ${SINGLEMDS}) --reformat \
242 ${MDT_DEVNAME} $(mdsvdevname 1) > /dev/null ||
243 error "Fail to reformat the MDS!"
245 echo "+++ start to create for ${i} files set at: $(date) +++"
246 lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
247 error "Fail to create files!"
248 echo "+++ end to create for ${i} files set at: $(date) +++"
250 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
251 error "Fail to start MDS!"
253 echo "start lfsck_namespace for ${i} files set at: $(date)"
254 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
257 local STATUS=$($SHOW_NAMESPACE |
258 awk '/^status/ { print $2 }')
259 [ "$STATUS" == "completed" ] && break
260 sleep 3 # check status every 3 seconds
263 echo "end lfsck_namespace for ${i} files set at: $(date)"
264 local SPEED=$($SHOW_NAMESPACE |
265 awk '/^average_speed_phase1/ { print $2 }')
266 echo "lfsck_namespace speed is ${SPEED}/sec"
267 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
270 run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
273 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
275 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
280 do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
281 reformat_external_journal
282 add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype \
283 $(facet_fstype ${SINGLEMDS}) --reformat ${MDT_DEVNAME} \
284 $(mdsvdevname 1) > /dev/null || error "Fail to reformat the MDS"
286 for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
287 local nfiles=$((i - BCOUNT))
289 echo "+++ start to create for ${i} files set at: $(date) +++"
290 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
291 error "Fail to create files!"
292 echo "+++ end to create for ${i} files set at: $(date) +++"
296 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
297 error "Fail to start MDS!"
299 echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
300 $START_NAMESPACE || error "Fail to start lfsck_namespace!"
303 local STATUS=$($SHOW_NAMESPACE |
304 awk '/^status/ { print $2 }')
305 [ "$STATUS" == "completed" ] && break
306 sleep 3 # check status every 3 seconds
309 echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
310 local FULL_SPEED=$($SHOW_NAMESPACE |
311 awk '/^average_speed_phase1/ { print $2 }')
312 echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
313 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
314 local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
317 for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
318 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
319 error "Fail to start MDS!"
321 $STOP_LFSCK > /dev/null 2>&1
322 echo "start lfsck_namespace with speed ${j} at: $(date)"
323 $START_NAMESPACE --reset -s ${j} ||
324 error "Fail to start lfsck_namespace with speed ${j}!"
325 # lfsck_namespace will be paused when MDS stop,
326 # and will be restarted automatically when mount up again.
327 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
329 local nfiles=$(((i - BCOUNT) / 2))
331 echo "+++ start to create for ${i} files set at: $(date) +++"
332 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
333 error "Fail to create files!"
334 echo "+++ end to create for ${i} files set at: $(date) +++"
339 start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
340 error "Fail to start MDS!"
342 $STOP_LFSCK /dev/null 2>&1
343 echo "start lfsck_namespace with full speed at: $(date)"
344 $START_NAMESPACE --reset -s 0 ||
345 error "Fail to start lfsck_namespace with full speed!"
346 stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
348 local nfiles=$(((i - BCOUNT) / 2))
350 echo "+++ start to create for ${i} files set at: $(date) +++"
351 lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
352 error "Fail to create files!"
353 echo "+++ end to create for ${i} files set at: $(date) +++"
355 run_test 3 "lfsck namespace impact on create performance"
361 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
365 echo "***** Start layout LFSCK on all devices at: $(date) *****"
366 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
368 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
369 mdd.${MDT_DEV}.lfsck_layout |
370 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
374 echo "***** End layout LFSCK on all devices at: $(date) *****"
376 for n in $(seq $MDSCOUNT); do
379 local SPEED=$(show_layout ${n} |
380 awk '/^average_speed_phase1/ { print $2 }')
382 echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
390 local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
391 local basedir="$mntpt/$tdir/$idx1/$idx2"
394 error_noexit "(11) Fail to mkdir $mntpt"
398 mount_client $mntpt || {
399 error_noexit "(12) Fail to mount $mntpt"
405 error_noexit "(13) Fail to mkdir $basedir"
409 echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
410 createmany -o ${basedir}/f $UNIT || {
412 error_noexit "(14) Fail to gen $UNIT files under $basedir"
415 echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
423 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
424 for ((k = 0; k < $MDSCOUNT; k++)); do
425 $LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
427 for ((l = 1; l <= $cnt; l++)); do
428 layout_gen_one ${k} ${l} &
433 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
437 local saved_mdscount=$MDSCOUNT
438 local saved_ostcount=$OSTCOUNT
441 stopall > /dev/null || error "(1) Fail to stopall"
443 LFSCKDIR="$DIR/$tdir"
445 for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
448 echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
451 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
453 formatall > /dev/null ||
454 error "(2) Fail to formatall, subdirs=${j}"
457 setupall > /dev/null ||
458 error "(3) Fail to setupall, subdirs=${j}"
461 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
463 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
464 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
467 layout_gen_set ${j} || RC=$?
469 error "(6) generate set $RC, subdirs=${j}"
472 layout_test_one || RC=$?
474 error "(7) LFSCK failed with $RC, subdirs=${j}"
478 stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
481 echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
484 MDSCOUNT=$saved_mdscount
485 OSTCOUNT=$saved_ostcount
491 run_test 4a "Single MDS lfsck layout performance (routine case) without load"
494 echo "Inject failure stub to simulate dangling reference"
495 #define OBD_FAIL_LFSCK_DANGLING 0x1610
496 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
500 run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
503 local saved_mdscount=$MDSCOUNT
506 stopall > /dev/null || error "(1) Fail to stopall"
508 LFSCKDIR="$DIR/$tdir"
509 for ((i = 1; i <= $saved_mdscount; i++)); do
512 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
515 for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
517 formatall > /dev/null ||
518 error "(2) Fail to formatall, subdirs=${j}"
521 setupall > /dev/null ||
522 error "(3) Fail to setupall, subdirs=${j}"
525 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
527 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
528 error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
531 layout_gen_set ${j} || RC=$?
533 error "(6) generate set $RC, subdirs=${j}"
536 layout_test_one || RC=$?
538 error "(7) LFSCK failed with $RC, subdirs=${j}"
542 stopall > /dev/null || error "(8) Fail to stopall"
545 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
548 MDSCOUNT=$saved_mdscount
554 run_test 5a "lfsck layout performance (routine case) without load for DNE"
557 echo "Inject failure stub to simulate dangling reference"
558 #define OBD_FAIL_LFSCK_DANGLING 0x1610
559 do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610
563 run_test 5b "lfsck layout performance (repairing case) without load for DNE"
565 lfsck_fast_create() {
569 local ldir="/test-${lbase}"
574 [ $count -eq 0 -o $count -gt ${total} ] && count=$total
575 local usize=$((count / NTHREADS))
576 [ ${usize} -eq 0 ] && break
577 local tdir=${ldir}-${cycle}-
579 echo "[cycle: $cycle] [threads: $threads]"\
580 "[files: $count] [basedir: $tdir]"
584 total=$((total - usize * NTHREADS))
585 [ $total -eq 0 ] && break
586 lbase=$((lbase + usize))
591 lfsck_detach_error() {
597 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
600 stopall > /dev/null || error "(1) Fail to stopall"
602 local saved_mdscount=$MDSCOUNT
604 LFSCKDIR="$DIR/$tdir"
607 formatall > /dev/null || error "(2) Fail to formatall"
610 setupall > /dev/null || error "(3) Fail to setupall"
612 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
614 $LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
615 error "(5) Fail to setstripe on $LFSCKDIR"
618 layout_gen_set $TOTSUBDIR || RC=$?
620 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
623 echo "***** Start layout LFSCK on single MDS at: $(date) *****"
624 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
625 error "(7) Fail to start layout LFSCK"
627 wait_update_facet $SINGLEMDS "$LCTL get_param -n \
628 mdd.${MDT_DEV}.lfsck_layout |
629 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
631 error "(8) layout LFSCK cannot finished in time"
633 echo "***** End layout LFSCK on single MDS at: $(date) *****"
635 local SPEED=$(show_layout 1 |
636 awk '/^average_speed_phase1/ { print $2 }')
637 echo "lfsck_layout full_speed is $SPEED objs/sec"
639 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
640 local nfiles=$((inc_count / 2))
643 for ((m = 0, n = $INCFACTOR; n < 100;
644 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
645 local sl=$((SPEED * n / 100))
647 $STOP_LFSCK > /dev/null 2>&1
649 echo "start lfsck_layout with speed ${sl} at: $(date)"
650 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
652 "(9) Fail to start lfsck_layout with speed ${sl}"
654 echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
655 lfsck_fast_create $nfiles ${m} $NTHREADS ||
656 lfsck_detach_error "(10) Fail to create files"
657 echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
660 $STOP_LFSCK > /dev/null 2>&1
662 echo "start lfsck_layout with full speed at: $(date)"
663 $RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
665 "(11) Fail to start lfsck_layout with full speed"
667 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
668 lfsck_fast_create $nfiles ${m} $NTHREADS ||
669 lfsck_detach_error "(12) Fail to create files"
670 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
673 $STOP_LFSCK > /dev/null 2>&1
675 echo "create without lfsck_layout run back-ground"
676 echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
677 lfsck_fast_create $nfiles ${m} $NTHREADS ||
678 lfsck_detach_error "(13) Fail to create files"
679 echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
684 stopall > /dev/null || error "(14) Fail to stopall"
686 MDSCOUNT=$saved_mdscount
688 run_test 6 "lfsck layout impact on create performance"
694 "$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
697 namespace_test_one() {
698 echo "***** Start namespace LFSCK on all devices at: $(date) *****"
699 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
701 for n in $(seq $MDSCOUNT); do
702 wait_update_facet mds${n} "$LCTL get_param -n \
703 mdd.$(facet_svc mds${n}).lfsck_namespace |
704 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
709 echo "***** End namespace LFSCK on all devices at: $(date) *****"
711 for n in $(seq $MDSCOUNT); do
714 local SPEED=$(show_namespace ${n} |
715 awk '/^average_speed_total/ { print $2 }')
717 echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
722 namespace_gen_one() {
725 local idx3=$(((idx1 + 1) % MDSCOUNT))
726 local base_mntpt="/mnt/lustre_lfsck_${idx1}"
727 local show_dir="$LFSCKDIR/${idx1}/${idx2}"
728 local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
730 mkdir $show_dir || return 20
732 local count=$((UNIT * 78 / 100)) # 78% regular files
733 local sub_count=$((count / NTHREADS))
734 echo "Creating $count regular files under $show_dir at: $(date)"
735 for ((m = 0; m < $NTHREADS; m++)); do
736 local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
738 createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
742 error_noexit "(21) Fail to gen regular files under $show_dir"
746 count=$((UNIT * 3 / 100)) # 3% local sub-dirs
747 echo "Creating $count local sub-dirs under $show_dir at: $(date)"
748 createmany -d $work_dir/d_l_ $count > /dev/null || {
749 error_noexit "(22) Fail to gen local sub-dir under $show_dir"
753 # 3% * 5 = 15% regular files under local sub-dirs
754 echo "Creating 5 regular files under each local sub-dir at: $(date)"
755 for ((m = 0; m < $count; m++)); do
756 createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
758 "(23) Fail to gen regular under $work_dir/d_l_${m}"
763 count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
764 echo "Creating $count multiple hard-links under $show_dir at: $(date)"
765 for ((m = 0; m < $count; m++)); do
766 ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
768 "(24) Fail to hardlink to $work_dir/f_0_${m}"
773 count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
774 echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
775 "files under each remote sub-dir at: $(date)"
776 for ((m = 0; m < $count; m++)); do
777 $LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
779 "(25) Fail to remote mkdir $work_dir/d_r_${m}"
783 # 0.3% * 4 = 1.2% regular files under remote sub-dirs
784 createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
786 "(26) Fail to gen regular under $work_dir/d_r_${m}"
791 # 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
792 count=$((UNIT * 3 / 1000))
793 echo "Creating $count 2-striped sub-dirs under $show_dir," \
794 "and 4 regular files under each striped sub-dir at: $(date)"
795 for ((m = 0; m < $count; m++)); do
796 $LFS setdirstripe -i ${idx1} -c 2 -t all_char \
797 $work_dir/d_s_${m} || {
799 "(27) Fail to make striped-dir $work_dir/d_s_${m}"
803 # 0.3% * 4 = 1.2% regular files under striped sub-dirs
804 createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
806 "(28) Fail to gen regular under $work_dir/d_s_${m}"
812 namespace_gen_mdt() {
815 local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
817 $LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
819 for ((m = 0; m < $NTHREADS; m++)); do
820 local mntpt="${base_mntpt}_${m}"
823 umount ${base_mntpt}_*
824 error_noexit "(11) Fail to mkdir $mntpt"
828 mount_client $mntpt || {
829 umount ${base_mntpt}_*
830 error_noexit "(12) Fail to mount $mntpt"
835 for ((l = 0; l < $dir_cnt; l++)); do
836 namespace_gen_one ${mdt_idx} ${l}
839 umount ${base_mntpt}_*
842 namespace_gen_set() {
846 echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
847 for ((k = 0; k < $MDSCOUNT; k++)); do
848 namespace_gen_mdt ${k} ${cnt} &
851 echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
856 local saved_mdscount=$MDSCOUNT
858 [ $MDSCOUNT -le 8 ] ||
859 error "Too much MDT, test data set on each MDT may be unbalance"
862 stopall > /dev/null || error "(1) Fail to stopall"
864 LFSCKDIR="$DIR/$tdir"
865 for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
868 echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
871 for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
872 j = $((j + MINSUBDIR)))); do
874 formatall > /dev/null ||
875 error "(2) Fail to formatall, subdirs=${j}"
878 setupall > /dev/null ||
879 error "(3) Fail to setupall, subdirs=${j}"
882 error "(4) mkdir $LFSCKDIR, subdirs=${j}"
884 $LFS setstripe -c 1 -i -1 $LFSCKDIR ||
885 error "(5) Fail to setstripe on $LFSCKDIR"
887 do_nodes $(comma_list $(mdts_nodes)) \
888 $LCTL set_param fail_loc=$local_loc
891 namespace_gen_set ${j} || RC=$?
893 error "(6) generate set $RC, subdirs=${j}"
896 namespace_test_one || RC=$?
898 error "(7) LFSCK failed with $RC, subdirs=${j}"
900 do_nodes $(comma_list $(mdts_nodes)) \
901 $LCTL set_param fail_loc=0
905 stopall > /dev/null || error "(8) Fail to stopall"
908 echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
911 MDSCOUNT=$saved_mdscount
917 run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
920 echo "Inject failure stub to simulate the case of lost linkEA"
921 #define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
924 run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
927 echo "Inject failure stub to simulate the case of bad FID-in-dirent"
928 #define OBD_FAIL_FID_INDIR 0x1501
931 run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
934 [ $MDSCOUNT -lt 2 ] &&
935 skip "We need at least 2 MDSes for this test" && return
937 [ $INCFACTOR -gt 25 ] && INCFACTOR=25
940 stopall > /dev/null || error "(1) Fail to stopall"
942 local saved_mdscount=$MDSCOUNT
944 LFSCKDIR="$DIR/$tdir"
947 formatall > /dev/null || error "(2) Fail to formatall"
950 setupall > /dev/null || error "(3) Fail to setupall"
952 mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
954 $LFS setstripe -c 1 -i 0 $LFSCKDIR ||
955 error "(5) Fail to setstripe on $LFSCKDIR"
958 namespace_gen_set $TOTSUBDIR || RC=$?
960 error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
963 echo "***** Start namespace LFSCK at: $(date) *****"
964 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
965 error "(7) Fail to start namespace LFSCK"
967 for n in $(seq $MDSCOUNT); do
968 wait_update_facet mds${n} "$LCTL get_param -n \
969 mdd.$(facet_svc mds${n}).lfsck_namespace |
970 awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
972 error "(8) namespace LFSCK cannot finished in time"
975 echo "***** End namespace LFSCK at: $(date) *****"
977 local SPEED=$(show_namespace 1 |
978 awk '/^average_speed_phase1/ { print $2 }')
979 echo "lfsck_namespace full_speed is $SPEED objs/sec"
982 local inc_count=$((BASE_COUNT * INCFACTOR / 100))
983 local nfiles=$((inc_count / 2))
988 local stime=$(date +%s)
989 lfsck_fast_create $nfiles ${m} $NTHREADS ||
990 lfsck_detach_error "(9) Fail to create files"
991 local etime=$(date +%s)
992 echo "created $nfiles without lfsck_namespace run back-ground used" \
993 "$((etime - stime)) seconds"
996 for ((m = nfiles, n = $INCFACTOR; n < 100;
997 m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
998 local sl=$((SPEED * n / 100))
1000 $STOP_LFSCK > /dev/null 2>&1
1001 echo "start lfsck_namespace with speed ${sl} at: $(date)"
1002 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
1003 lfsck_detach_error \
1004 "(10) Fail to start lfsck_namespace with speed ${sl}"
1007 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1008 lfsck_detach_error "(11) Fail to create files"
1010 echo "created $nfiles with namespace LFSCK run with the" \
1011 "speed limit of ${n}% of full speed used" \
1012 "$((etime - stime)) seconds"
1016 $STOP_LFSCK > /dev/null 2>&1
1017 echo "start lfsck_namespace with full speed at: $(date)"
1018 $RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
1019 lfsck_detach_error \
1020 "(12) Fail to start lfsck_namespace with full speed"
1023 lfsck_fast_create $nfiles ${m} $NTHREADS ||
1024 lfsck_detach_error "(13) Fail to create files"
1026 echo "created $nfiles with namespace LFSCK run with full speed used" \
1027 "$((etime - stime)) seconds"
1030 $STOP_LFSCK > /dev/null 2>&1
1035 stopall > /dev/null || error "(14) Fail to stopall"
1037 MDSCOUNT=$saved_mdscount
1039 run_test 8 "lfsck namespace impact on create performance"
1041 # cleanup the system at last